From 6fb06464817c127a07a7ea6aeeec3f37768238ad Mon Sep 17 00:00:00 2001 From: Matt Gibson Date: Fri, 4 Dec 2020 12:29:31 -0600 Subject: [PATCH] Fix 1password importer (#217) * Fix import of 1password csv * 1password is using '\' as a quote escape character. * 1password's csv headers are sometimes capitalized. We want to identify them case insensitively * Change cipher type based on csv type header * Translate 1password data to correct fields * Test identity and credit card import * linter fixes * Do not use node 'fs' module Karma is being used for automated tests so node modules are not available Co-authored-by: Matt Gibson --- .../importers/onepasswordCsvImporter.spec.ts | 50 +++++++++++ .../testData/onePasswordCsv/creditCard.csv.ts | 3 + .../testData/onePasswordCsv/identity.csv.ts | 6 ++ src/importers/baseImporter.ts | 14 +-- src/importers/onepasswordWinCsvImporter.ts | 89 ++++++++++++++++--- 5 files changed, 143 insertions(+), 19 deletions(-) create mode 100644 spec/common/importers/onepasswordCsvImporter.spec.ts create mode 100644 spec/common/importers/testData/onePasswordCsv/creditCard.csv.ts create mode 100644 spec/common/importers/testData/onePasswordCsv/identity.csv.ts diff --git a/spec/common/importers/onepasswordCsvImporter.spec.ts b/spec/common/importers/onepasswordCsvImporter.spec.ts new file mode 100644 index 0000000000..299b4901ed --- /dev/null +++ b/spec/common/importers/onepasswordCsvImporter.spec.ts @@ -0,0 +1,50 @@ +import { OnePasswordWinCsvImporter as Importer } from '../../../src/importers/onepasswordWinCsvImporter'; + +import { CipherType } from '../../../src/enums'; + +import { data as creditCardData } from './testData/onePasswordCsv/creditCard.csv' +import { data as identityData } from './testData/onePasswordCsv/identity.csv' + +describe('1Password CSV Importer', () => { + it('should parse identity imports', () => { + const importer = new Importer(); + const result = importer.parse(identityData); + + expect(result).not.toBeNull(); + expect(result.success).toBe(true); + expect(result.ciphers.length).toBe(1); + const cipher = result.ciphers[0]; + expect(cipher.type).toBe(CipherType.Identity) + + expect(cipher.identity).toEqual(jasmine.objectContaining({ + firstName: 'first name', + middleName: 'mi', + lastName: 'last name', + username: 'userNam3', + company: 'bitwarden', + phone: '8005555555', + email: 'email@bitwarden.com' + })); + + expect(cipher.notes).toContain('address\ncity state zip\nUnited States'); + }); + + it('should parse credit card imports', () => { + const importer = new Importer(); + const result = importer.parse(creditCardData); + + expect(result).not.toBeNull(); + expect(result.success).toBe(true); + expect(result.ciphers.length).toBe(1); + const cipher = result.ciphers[0]; + expect(cipher.type).toBe(CipherType.Card); + + expect(cipher.card).toEqual(jasmine.objectContaining({ + number: '4111111111111111', + code: '111', + cardholderName: 'test', + expMonth: '1', + expYear: '2030', + })); + }); +}); diff --git a/spec/common/importers/testData/onePasswordCsv/creditCard.csv.ts b/spec/common/importers/testData/onePasswordCsv/creditCard.csv.ts new file mode 100644 index 0000000000..47c05222a5 --- /dev/null +++ b/spec/common/importers/testData/onePasswordCsv/creditCard.csv.ts @@ -0,0 +1,3 @@ +export const data = `"account number(accountNo)","address(address)","address(branchAddress)","admin console URL(admin_console_url)","admin console username(admin_console_username)","AirPort ID(airport_id)","alias(alias)","AOL/AIM(aim)","approved wildlife(game)","attached storage password(disk_password)","auth​ method(pop_authentication)","auth​ method(smtp_authentication)","bank name(bankName)","base station name(name)","base station password(password)","birth date(birthdate)","business(busphone)","cardholder name(cardholder)","cash withdrawal limit(cashLimit)","cell(cellphone)","company name(company_name)","company(company)","conditions / restrictions(conditions)","connection options(options)","console password(admin_console_password)","country(country)","Created Date","credit limit(creditLimit)","customer service phone(customer_service_phone)","database(database)","date of birth(birthdate)","default phone(defphone)","department(department)","download page(download_link)","email(email)","expires(expires)","expiry date(expiry_date)","expiry date(expiry)","first name(firstname)","forum signature(forumsig)","full name(fullname)","full name(name)","group(org_name)","height(height)","home(homephone)","IBAN(iban)","ICQ(icq)","initial(initial)","interest rate(interest)","issue number(issuenumber)","issued on(issue_date)","issuing authority(issuing_authority)","issuing bank(bank)","issuing country(issuing_country)","job title(jobtitle)","last name(lastname)","license class(class)","license key(reg_code)","licensed to(reg_name)","maximum quota(quota)","member ID (additional)(additional_no)","member ID(membership_no)","member name(member_name)","member since(member_since)","Modified Date","MSN(msn)","name on account(owner)","name(name)","nationality(nationality)","network name(network_name)","Notes","number(ccnum)","number(number)","occupation(occupation)","order number(order_number)","order total(order_total)","Password","password(password)","password(pop_password)","password(smtp_password)","phone (intl)(phoneIntl)","phone (local)(phone_local)","phone (local)(phoneLocal)","phone (toll free)(phone_tollfree)","phone (toll free)(phoneTollFree)","phone for reserva​tions(reservations_phone)","phone(branchPhone)","PIN(pin)","PIN(telephonePin)","place of birth(birthplace)","port number(pop_port)","port number(smtp_port)","port(port)","provider's website(provider_website)","provider(provider)","publisher(publisher_name)","purchase date(order_date)","registered email(reg_email)","reminder answer(remindera)","reminder question(reminderq)","retail price(retail_price)","routing number(routingNo)","Scope","security(pop_security)","security(smtp_security)","server / IP address(server)","server(hostname)","server(pop_server)","sex(sex)","SID(sid)","skype(skype)","SMTP server(smtp_server)","state(state)","support email(support_email)","support phone(support_contact_phone)","support URL(support_contact_url)","SWIFT(swift)","Tags","telephone(phone)","Title","Type","type(accountType)","type(database_type)","type(pop_type)","type(type)","URL","URL(url)","Username","username(pop_username)","username(smtp_username)","username(username)","valid from(valid_from)","valid from(validFrom)","verification number(cvv)","version(product_version)","website(publisher_website)","website(website)","wireless network password(wireless_password)","wireless security(wireless_security)","Yahoo(yahoo)", +,,,,,,,,,,,,,,,,,"test",,,,,,,,,"1606923869",,,,,,,,,,,"01/2030",,,,,,,,,,,,,,,,,,,,,,,,,,,"1606924056",,,,,,"","4111111111111111",,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,"{( +)}",,"test card","Credit Card",,,,"laser",,,,,,,,,"111",,,,,,,` diff --git a/spec/common/importers/testData/onePasswordCsv/identity.csv.ts b/spec/common/importers/testData/onePasswordCsv/identity.csv.ts new file mode 100644 index 0000000000..2b74cd8d8a --- /dev/null +++ b/spec/common/importers/testData/onePasswordCsv/identity.csv.ts @@ -0,0 +1,6 @@ +export const data = `"account number(accountNo)","address(address)","address(branchAddress)","admin console URL(admin_console_url)","admin console username(admin_console_username)","AirPort ID(airport_id)","alias(alias)","AOL/AIM(aim)","approved wildlife(game)","attached storage password(disk_password)","auth​ method(pop_authentication)","auth​ method(smtp_authentication)","bank name(bankName)","base station name(name)","base station password(password)","birth date(birthdate)","business(busphone)","cardholder name(cardholder)","cash withdrawal limit(cashLimit)","cell(cellphone)","company name(company_name)","company(company)","conditions / restrictions(conditions)","connection options(options)","console password(admin_console_password)","country(country)","Created Date","credit limit(creditLimit)","customer service phone(customer_service_phone)","database(database)","date of birth(birthdate)","default phone(defphone)","department(department)","download page(download_link)","email(email)","expires(expires)","expiry date(expiry_date)","expiry date(expiry)","first name(firstname)","forum signature(forumsig)","full name(fullname)","full name(name)","group(org_name)","height(height)","home(homephone)","IBAN(iban)","ICQ(icq)","initial(initial)","interest rate(interest)","issue number(issuenumber)","issued on(issue_date)","issuing authority(issuing_authority)","issuing bank(bank)","issuing country(issuing_country)","job title(jobtitle)","last name(lastname)","license class(class)","license key(reg_code)","licensed to(reg_name)","maximum quota(quota)","member ID (additional)(additional_no)","member ID(membership_no)","member name(member_name)","member since(member_since)","Modified Date","MSN(msn)","name on account(owner)","name(name)","nationality(nationality)","network name(network_name)","Notes","number(ccnum)","number(number)","occupation(occupation)","order number(order_number)","order total(order_total)","Password","password(password)","password(pop_password)","password(smtp_password)","phone (intl)(phoneIntl)","phone (local)(phone_local)","phone (local)(phoneLocal)","phone (toll free)(phone_tollfree)","phone (toll free)(phoneTollFree)","phone for reserva​tions(reservations_phone)","phone(branchPhone)","PIN(pin)","PIN(telephonePin)","place of birth(birthplace)","port number(pop_port)","port number(smtp_port)","port(port)","provider's website(provider_website)","provider(provider)","publisher(publisher_name)","purchase date(order_date)","registered email(reg_email)","reminder answer(remindera)","reminder question(reminderq)","retail price(retail_price)","routing number(routingNo)","Scope","security(pop_security)","security(smtp_security)","server / IP address(server)","server(hostname)","server(pop_server)","sex(sex)","SID(sid)","skype(skype)","SMTP server(smtp_server)","state(state)","support email(support_email)","support phone(support_contact_phone)","support URL(support_contact_url)","SWIFT(swift)","Tags","telephone(phone)","Title","Type","type(accountType)","type(database_type)","type(pop_type)","type(type)","URL","URL(url)","Username","username(pop_username)","username(smtp_username)","username(username)","valid from(valid_from)","valid from(validFrom)","verification number(cvv)","version(product_version)","website(publisher_website)","website(website)","wireless network password(wireless_password)","wireless security(wireless_security)","Yahoo(yahoo)", +,"address +city state zip +United States",,,,,,"",,,,,,,,"12/2/20","",,,"",,"bitwarden",,,,,"1606923754",,,,"12/2/20","8005555555","department",,"email@bitwarden.com",,,,"first name","",,,,,"",,"","mi",,,,,,,"job title","last name",,,,,,,,,"1607020883","",,,,,"It’s you! 🖐 Select Edit to fill in more details, like your address and contact information.",,,"occupation",,,,,,,,,,,,,,,,,,,,,,,,,"","",,,,,,,,,"",,"",,,,,,,"{( + \\"Starter Kit\\" +)}",,"Identity Item","Identity",,,,,,,"userNam3",,,"userNam3",,,,,,"",,,"",` diff --git a/src/importers/baseImporter.ts b/src/importers/baseImporter.ts index 9c0ce1eef0..2999227d3c 100644 --- a/src/importers/baseImporter.ts +++ b/src/importers/baseImporter.ts @@ -65,19 +65,21 @@ export abstract class BaseImporter { 'ort', 'adresse', ]; + protected parseCsvOptions = { + encoding: 'UTF-8', + skipEmptyLines: false, + } + protected parseXml(data: string): Document { const parser = new DOMParser(); const doc = parser.parseFromString(data, 'application/xml'); return doc != null && doc.querySelector('parsererror') == null ? doc : null; } - protected parseCsv(data: string, header: boolean): any[] { + protected parseCsv(data: string, header: boolean, options: any = {}): any[] { + const parseOptions = Object.assign({ header: header }, this.parseCsvOptions, options); data = this.splitNewLine(data).join('\n').trim(); - const result = papa.parse(data, { - header: header, - encoding: 'UTF-8', - skipEmptyLines: false, - }); + const result = papa.parse(data, parseOptions); if (result.errors != null && result.errors.length > 0) { result.errors.forEach((e) => { if (e.row != null) { diff --git a/src/importers/onepasswordWinCsvImporter.ts b/src/importers/onepasswordWinCsvImporter.ts index 0ca9a93ab1..110268da92 100644 --- a/src/importers/onepasswordWinCsvImporter.ts +++ b/src/importers/onepasswordWinCsvImporter.ts @@ -4,14 +4,17 @@ import { Importer } from './importer'; import { ImportResult } from '../models/domain/importResult'; import { CipherType } from '../enums/cipherType'; -import { CardView } from '../models/view'; +import { CardView, IdentityView } from '../models/view'; -const IgnoredProperties = ['ainfo', 'autosubmit', 'notesplain', 'ps', 'scope', 'tags', 'title', 'uuid']; +const IgnoredProperties = ['ainfo', 'autosubmit', 'notesplain', 'ps', 'scope', 'tags', 'title', 'uuid', 'notes']; export class OnePasswordWinCsvImporter extends BaseImporter implements Importer { parse(data: string): ImportResult { const result = new ImportResult(); - const results = this.parseCsv(data, true); + const results = this.parseCsv(data, true, { + quoteChar: '"', + escapeChar: '\\', + }); if (results == null) { result.success = false; return result; @@ -24,7 +27,29 @@ export class OnePasswordWinCsvImporter extends BaseImporter implements Importer const cipher = this.initLoginCipher(); cipher.name = this.getValueOrDefault(this.getProp(value, 'title'), '--'); - cipher.notes = this.getValueOrDefault(this.getProp(value, 'notesPlain'), '') + '\n'; + + cipher.notes = this.getValueOrDefault(this.getProp(value, 'notesPlain'), '') + '\n' + + this.getValueOrDefault(this.getProp(value, 'notes'), '') + '\n'; + cipher.notes.trim(); + + const onePassType = this.getValueOrDefault(this.getProp(value, 'type'), 'Login') + switch (onePassType) { + case 'Credit Card': + cipher.type = CipherType.Card; + cipher.card = new CardView(); + IgnoredProperties.push('type'); + break; + case 'Identity': + cipher.type = CipherType.Identity; + cipher.identity = new IdentityView(); + IgnoredProperties.push('type'); + break; + case 'Login': + case 'Secure Note': + IgnoredProperties.push('type'); + default: + break; + } if (!this.isNullOrWhitespace(this.getProp(value, 'number')) && !this.isNullOrWhitespace(this.getProp(value, 'expiry date'))) { @@ -50,30 +75,59 @@ export class OnePasswordWinCsvImporter extends BaseImporter implements Importer const urls = value[property].split(this.newLineRegex); cipher.login.uris = this.makeUriArray(urls); continue; + } else if ((lowerProp === 'url')) { + if (cipher.login.uris == null) { + cipher.login.uris = []; + } + cipher.login.uris.concat(this.makeUriArray(value[property])); + continue; } } else if (cipher.type === CipherType.Card) { - if (this.isNullOrWhitespace(cipher.card.number) && lowerProp === 'number') { + if (this.isNullOrWhitespace(cipher.card.number) && lowerProp.includes('number')) { cipher.card.number = value[property]; cipher.card.brand = this.getCardBrand(this.getProp(value, 'number')); continue; - } else if (this.isNullOrWhitespace(cipher.card.code) && lowerProp === 'verification number') { + } else if (this.isNullOrWhitespace(cipher.card.code) && lowerProp.includes('verification number')) { cipher.card.code = value[property]; continue; - } else if (this.isNullOrWhitespace(cipher.card.cardholderName) && lowerProp === 'cardholder name') { + } else if (this.isNullOrWhitespace(cipher.card.cardholderName) && lowerProp.includes('cardholder name')) { cipher.card.cardholderName = value[property]; continue; - } else if (this.isNullOrWhitespace(cipher.card.expiration) && lowerProp === 'expiry date' && - value[property].length === 6) { - cipher.card.expMonth = (value[property] as string).substr(4, 2); + } else if (this.isNullOrWhitespace(cipher.card.expiration) && lowerProp.includes('expiry date') && + value[property].length === 7) { + cipher.card.expMonth = (value[property] as string).substr(0, 2); if (cipher.card.expMonth[0] === '0') { cipher.card.expMonth = cipher.card.expMonth.substr(1, 1); } - cipher.card.expYear = (value[property] as string).substr(0, 4); + cipher.card.expYear = (value[property] as string).substr(3, 4); continue; - } else if (lowerProp === 'type') { + } else if (lowerProp === 'type' || lowerProp === 'type(type)') { // Skip since brand was determined from number above continue; } + } else if (cipher.type === CipherType.Identity) { + if (this.isNullOrWhitespace(cipher.identity.firstName) && lowerProp.includes('first name')) { + cipher.identity.firstName = value[property]; + continue; + } else if (this.isNullOrWhitespace(cipher.identity.middleName) && lowerProp.includes('initial')) { + cipher.identity.middleName = value[property]; + continue; + } else if (this.isNullOrWhitespace(cipher.identity.lastName) && lowerProp.includes('last name')) { + cipher.identity.lastName = value[property]; + continue; + } else if (this.isNullOrWhitespace(cipher.identity.username) && lowerProp.includes('username')) { + cipher.identity.username = value[property]; + continue; + } else if (this.isNullOrWhitespace(cipher.identity.company) && lowerProp.includes('company')) { + cipher.identity.company = value[property]; + continue; + } else if (this.isNullOrWhitespace(cipher.identity.phone) && lowerProp.includes('default phone')) { + cipher.identity.phone = value[property]; + continue; + } else if (this.isNullOrWhitespace(cipher.identity.email) && lowerProp.includes('email')) { + cipher.identity.email = value[property]; + continue; + } } if (IgnoredProperties.indexOf(lowerProp) === -1 && !lowerProp.startsWith('section:') && @@ -81,6 +135,11 @@ export class OnePasswordWinCsvImporter extends BaseImporter implements Importer if (altUsername == null && lowerProp === 'email') { altUsername = value[property]; } + else if (lowerProp === 'created date' || lowerProp === 'modified date') { + const readableDate = new Date(parseInt(value[property], 10) * 1000).toUTCString(); + this.processKvp(cipher, '1Password ' + property, readableDate); + continue; + } this.processKvp(cipher, property, value[property]); } } @@ -100,6 +159,10 @@ export class OnePasswordWinCsvImporter extends BaseImporter implements Importer } private getProp(obj: any, name: string): any { - return obj[name] || obj[name.toUpperCase()]; + const lowerObj = Object.entries(obj).reduce((agg: any, entry: [string, any]) => { + agg[entry[0].toLowerCase()] = entry[1]; + return agg; + }, {}); + return lowerObj[name.toLowerCase()]; } }