Fix 1password importer (#217)

* Fix import of 1password csv

* 1password is using '\' as a quote escape character.

* 1password's csv headers are sometimes capitalized. We want to identify
them case insensitively

* Change cipher type based on csv type header

* Translate 1password data to correct fields

* Test identity and credit card import

* linter fixes

* Do not use node 'fs' module

Karma is being used for automated tests so node modules are not available

Co-authored-by: Matt Gibson <mdgibson@Matts-MBP.lan>

spec/common/importers/onepasswordCsvImporter.spec.ts

@@ -0,0 +1,50 @@
+import { OnePasswordWinCsvImporter as Importer } from '../../../src/importers/onepasswordWinCsvImporter';
+
+import { CipherType } from '../../../src/enums';
+
+import { data as creditCardData } from './testData/onePasswordCsv/creditCard.csv'
+import { data as identityData } from './testData/onePasswordCsv/identity.csv'
+
+describe('1Password CSV Importer', () => {
+    it('should parse identity imports', () => {
+        const importer = new Importer();
+        const result = importer.parse(identityData);
+
+        expect(result).not.toBeNull();
+        expect(result.success).toBe(true);
+        expect(result.ciphers.length).toBe(1);
+        const cipher = result.ciphers[0];
+        expect(cipher.type).toBe(CipherType.Identity)
+
+        expect(cipher.identity).toEqual(jasmine.objectContaining({
+            firstName: 'first name',
+            middleName: 'mi',
+            lastName: 'last name',
+            username: 'userNam3',
+            company: 'bitwarden',
+            phone: '8005555555',
+            email: 'email@bitwarden.com'
+        }));
+
+        expect(cipher.notes).toContain('address\ncity state zip\nUnited States');
+    });
+
+    it('should parse credit card imports', () => {
+        const importer = new Importer();
+        const result = importer.parse(creditCardData);
+
+        expect(result).not.toBeNull();
+        expect(result.success).toBe(true);
+        expect(result.ciphers.length).toBe(1);
+        const cipher = result.ciphers[0];
+        expect(cipher.type).toBe(CipherType.Card);
+
+        expect(cipher.card).toEqual(jasmine.objectContaining({
+            number: '4111111111111111',
+            code: '111',
+            cardholderName: 'test',
+            expMonth: '1',
+            expYear: '2030',
+        }));
+    });
+});

spec/common/importers/testData/onePasswordCsv/creditCard.csv.ts

@@ -0,0 +1,3 @@
+export const data = `"account number(accountNo)","address(address)","address(branchAddress)","admin console URL(admin_console_url)","admin console username(admin_console_username)","AirPort ID(airport_id)","alias(alias)","AOL/AIM(aim)","approved wildlife(game)","attached storage password(disk_password)","auth​ method(pop_authentication)","auth​ method(smtp_authentication)","bank name(bankName)","base station name(name)","base station password(password)","birth date(birthdate)","business(busphone)","cardholder name(cardholder)","cash withdrawal limit(cashLimit)","cell(cellphone)","company name(company_name)","company(company)","conditions / restrictions(conditions)","connection options(options)","console password(admin_console_password)","country(country)","Created Date","credit limit(creditLimit)","customer service phone(customer_service_phone)","database(database)","date of birth(birthdate)","default phone(defphone)","department(department)","download page(download_link)","email(email)","expires(expires)","expiry date(expiry_date)","expiry date(expiry)","first name(firstname)","forum signature(forumsig)","full name(fullname)","full name(name)","group(org_name)","height(height)","home(homephone)","IBAN(iban)","ICQ(icq)","initial(initial)","interest rate(interest)","issue number(issuenumber)","issued on(issue_date)","issuing authority(issuing_authority)","issuing bank(bank)","issuing country(issuing_country)","job title(jobtitle)","last name(lastname)","license class(class)","license key(reg_code)","licensed to(reg_name)","maximum quota(quota)","member ID (additional)(additional_no)","member ID(membership_no)","member name(member_name)","member since(member_since)","Modified Date","MSN(msn)","name on account(owner)","name(name)","nationality(nationality)","network name(network_name)","Notes","number(ccnum)","number(number)","occupation(occupation)","order number(order_number)","order total(order_total)","Password","password(password)","password(pop_password)","password(smtp_password)","phone (intl)(phoneIntl)","phone (local)(phone_local)","phone (local)(phoneLocal)","phone (toll free)(phone_tollfree)","phone (toll free)(phoneTollFree)","phone for reserva​tions(reservations_phone)","phone(branchPhone)","PIN(pin)","PIN(telephonePin)","place of birth(birthplace)","port number(pop_port)","port number(smtp_port)","port(port)","provider's website(provider_website)","provider(provider)","publisher(publisher_name)","purchase date(order_date)","registered email(reg_email)","reminder answer(remindera)","reminder question(reminderq)","retail price(retail_price)","routing number(routingNo)","Scope","security(pop_security)","security(smtp_security)","server / IP address(server)","server(hostname)","server(pop_server)","sex(sex)","SID(sid)","skype(skype)","SMTP server(smtp_server)","state(state)","support email(support_email)","support phone(support_contact_phone)","support URL(support_contact_url)","SWIFT(swift)","Tags","telephone(phone)","Title","Type","type(accountType)","type(database_type)","type(pop_type)","type(type)","URL","URL(url)","Username","username(pop_username)","username(smtp_username)","username(username)","valid from(valid_from)","valid from(validFrom)","verification number(cvv)","version(product_version)","website(publisher_website)","website(website)","wireless network password(wireless_password)","wireless security(wireless_security)","Yahoo(yahoo)",
+,,,,,,,,,,,,,,,,,"test",,,,,,,,,"1606923869",,,,,,,,,,,"01/2030",,,,,,,,,,,,,,,,,,,,,,,,,,,"1606924056",,,,,,"","4111111111111111",,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,"{(
+)}",,"test card","Credit Card",,,,"laser",,,,,,,,,"111",,,,,,,`

spec/common/importers/testData/onePasswordCsv/identity.csv.ts

@@ -0,0 +1,6 @@
+export const data = `"account number(accountNo)","address(address)","address(branchAddress)","admin console URL(admin_console_url)","admin console username(admin_console_username)","AirPort ID(airport_id)","alias(alias)","AOL/AIM(aim)","approved wildlife(game)","attached storage password(disk_password)","auth​ method(pop_authentication)","auth​ method(smtp_authentication)","bank name(bankName)","base station name(name)","base station password(password)","birth date(birthdate)","business(busphone)","cardholder name(cardholder)","cash withdrawal limit(cashLimit)","cell(cellphone)","company name(company_name)","company(company)","conditions / restrictions(conditions)","connection options(options)","console password(admin_console_password)","country(country)","Created Date","credit limit(creditLimit)","customer service phone(customer_service_phone)","database(database)","date of birth(birthdate)","default phone(defphone)","department(department)","download page(download_link)","email(email)","expires(expires)","expiry date(expiry_date)","expiry date(expiry)","first name(firstname)","forum signature(forumsig)","full name(fullname)","full name(name)","group(org_name)","height(height)","home(homephone)","IBAN(iban)","ICQ(icq)","initial(initial)","interest rate(interest)","issue number(issuenumber)","issued on(issue_date)","issuing authority(issuing_authority)","issuing bank(bank)","issuing country(issuing_country)","job title(jobtitle)","last name(lastname)","license class(class)","license key(reg_code)","licensed to(reg_name)","maximum quota(quota)","member ID (additional)(additional_no)","member ID(membership_no)","member name(member_name)","member since(member_since)","Modified Date","MSN(msn)","name on account(owner)","name(name)","nationality(nationality)","network name(network_name)","Notes","number(ccnum)","number(number)","occupation(occupation)","order number(order_number)","order total(order_total)","Password","password(password)","password(pop_password)","password(smtp_password)","phone (intl)(phoneIntl)","phone (local)(phone_local)","phone (local)(phoneLocal)","phone (toll free)(phone_tollfree)","phone (toll free)(phoneTollFree)","phone for reserva​tions(reservations_phone)","phone(branchPhone)","PIN(pin)","PIN(telephonePin)","place of birth(birthplace)","port number(pop_port)","port number(smtp_port)","port(port)","provider's website(provider_website)","provider(provider)","publisher(publisher_name)","purchase date(order_date)","registered email(reg_email)","reminder answer(remindera)","reminder question(reminderq)","retail price(retail_price)","routing number(routingNo)","Scope","security(pop_security)","security(smtp_security)","server / IP address(server)","server(hostname)","server(pop_server)","sex(sex)","SID(sid)","skype(skype)","SMTP server(smtp_server)","state(state)","support email(support_email)","support phone(support_contact_phone)","support URL(support_contact_url)","SWIFT(swift)","Tags","telephone(phone)","Title","Type","type(accountType)","type(database_type)","type(pop_type)","type(type)","URL","URL(url)","Username","username(pop_username)","username(smtp_username)","username(username)","valid from(valid_from)","valid from(validFrom)","verification number(cvv)","version(product_version)","website(publisher_website)","website(website)","wireless network password(wireless_password)","wireless security(wireless_security)","Yahoo(yahoo)",
+,"address
+city state zip
+United States",,,,,,"",,,,,,,,"12/2/20","",,,"",,"bitwarden",,,,,"1606923754",,,,"12/2/20","8005555555","department",,"email@bitwarden.com",,,,"first name","",,,,,"",,"","mi",,,,,,,"job title","last name",,,,,,,,,"1607020883","",,,,,"It’s you! 🖐 Select Edit to fill in more details, like your address and contact information.",,,"occupation",,,,,,,,,,,,,,,,,,,,,,,,,"","",,,,,,,,,"",,"",,,,,,,"{(
+    \\"Starter Kit\\"
+)}",,"Identity Item","Identity",,,,,,,"userNam3",,,"userNam3",,,,,,"",,,"",`

src/importers/baseImporter.ts

@@ -65,19 +65,21 @@ export abstract class BaseImporter {
         'ort', 'adresse',
     ];
 
+    protected parseCsvOptions = {
+        encoding: 'UTF-8',
+        skipEmptyLines: false,
+    }
+
     protected parseXml(data: string): Document {
         const parser = new DOMParser();
         const doc = parser.parseFromString(data, 'application/xml');
         return doc != null && doc.querySelector('parsererror') == null ? doc : null;
     }
 
-    protected parseCsv(data: string, header: boolean): any[] {
+    protected parseCsv(data: string, header: boolean, options: any = {}): any[] {
+        const parseOptions = Object.assign({ header: header }, this.parseCsvOptions, options);
         data = this.splitNewLine(data).join('\n').trim();
-        const result = papa.parse(data, {
+        const result = papa.parse(data, parseOptions);
-            header: header,
-            encoding: 'UTF-8',
-            skipEmptyLines: false,
-        });
         if (result.errors != null && result.errors.length > 0) {
             result.errors.forEach((e) => {
                 if (e.row != null) {

src/importers/onepasswordWinCsvImporter.ts

@@ -4,14 +4,17 @@ import { Importer } from './importer';
 import { ImportResult } from '../models/domain/importResult';
 
 import { CipherType } from '../enums/cipherType';
-import { CardView } from '../models/view';
+import { CardView, IdentityView } from '../models/view';
 
-const IgnoredProperties = ['ainfo', 'autosubmit', 'notesplain', 'ps', 'scope', 'tags', 'title', 'uuid'];
+const IgnoredProperties = ['ainfo', 'autosubmit', 'notesplain', 'ps', 'scope', 'tags', 'title', 'uuid', 'notes'];
 
 export class OnePasswordWinCsvImporter extends BaseImporter implements Importer {
     parse(data: string): ImportResult {
         const result = new ImportResult();
-        const results = this.parseCsv(data, true);
+        const results = this.parseCsv(data, true, {
+            quoteChar: '"',
+            escapeChar: '\\',
+        });
         if (results == null) {
             result.success = false;
             return result;
@@ -24,7 +27,29 @@ export class OnePasswordWinCsvImporter extends BaseImporter implements Importer
 
             const cipher = this.initLoginCipher();
             cipher.name = this.getValueOrDefault(this.getProp(value, 'title'), '--');
-            cipher.notes = this.getValueOrDefault(this.getProp(value, 'notesPlain'), '') + '\n';
+
+            cipher.notes = this.getValueOrDefault(this.getProp(value, 'notesPlain'), '') + '\n' +
+                this.getValueOrDefault(this.getProp(value, 'notes'), '') + '\n';
+            cipher.notes.trim();
+
+            const onePassType = this.getValueOrDefault(this.getProp(value, 'type'), 'Login')
+            switch (onePassType) {
+                case 'Credit Card':
+                    cipher.type = CipherType.Card;
+                    cipher.card = new CardView();
+                    IgnoredProperties.push('type');
+                    break;
+                case 'Identity':
+                    cipher.type = CipherType.Identity;
+                    cipher.identity = new IdentityView();
+                    IgnoredProperties.push('type');
+                    break;
+                case 'Login':
+                case 'Secure Note':
+                    IgnoredProperties.push('type');
+                default:
+                    break;
+            }
 
             if (!this.isNullOrWhitespace(this.getProp(value, 'number')) &&
                 !this.isNullOrWhitespace(this.getProp(value, 'expiry date'))) {
@@ -50,30 +75,59 @@ export class OnePasswordWinCsvImporter extends BaseImporter implements Importer
                         const urls = value[property].split(this.newLineRegex);
                         cipher.login.uris = this.makeUriArray(urls);
                         continue;
+                    } else if ((lowerProp === 'url')) {
+                        if (cipher.login.uris == null) {
+                            cipher.login.uris = [];
+                        }
+                        cipher.login.uris.concat(this.makeUriArray(value[property]));
+                        continue;
                     }
                 } else if (cipher.type === CipherType.Card) {
-                    if (this.isNullOrWhitespace(cipher.card.number) && lowerProp === 'number') {
+                    if (this.isNullOrWhitespace(cipher.card.number) && lowerProp.includes('number')) {
                         cipher.card.number = value[property];
                         cipher.card.brand = this.getCardBrand(this.getProp(value, 'number'));
                         continue;
-                    } else if (this.isNullOrWhitespace(cipher.card.code) && lowerProp === 'verification number') {
+                    } else if (this.isNullOrWhitespace(cipher.card.code) && lowerProp.includes('verification number')) {
                         cipher.card.code = value[property];
                         continue;
-                    } else if (this.isNullOrWhitespace(cipher.card.cardholderName) && lowerProp === 'cardholder name') {
+                    } else if (this.isNullOrWhitespace(cipher.card.cardholderName) && lowerProp.includes('cardholder name')) {
                         cipher.card.cardholderName = value[property];
                         continue;
-                    } else if (this.isNullOrWhitespace(cipher.card.expiration) && lowerProp === 'expiry date' &&
+                    } else if (this.isNullOrWhitespace(cipher.card.expiration) && lowerProp.includes('expiry date') &&
-                        value[property].length === 6) {
+                        value[property].length === 7) {
-                        cipher.card.expMonth = (value[property] as string).substr(4, 2);
+                        cipher.card.expMonth = (value[property] as string).substr(0, 2);
                         if (cipher.card.expMonth[0] === '0') {
                             cipher.card.expMonth = cipher.card.expMonth.substr(1, 1);
                         }
-                        cipher.card.expYear = (value[property] as string).substr(0, 4);
+                        cipher.card.expYear = (value[property] as string).substr(3, 4);
                         continue;
-                    } else if (lowerProp === 'type') {
+                    } else if (lowerProp === 'type' || lowerProp === 'type(type)') {
                         // Skip since brand was determined from number above
                         continue;
                     }
+                } else if (cipher.type === CipherType.Identity) {
+                    if (this.isNullOrWhitespace(cipher.identity.firstName) && lowerProp.includes('first name')) {
+                        cipher.identity.firstName = value[property];
+                        continue;
+                    } else if (this.isNullOrWhitespace(cipher.identity.middleName) && lowerProp.includes('initial')) {
+                        cipher.identity.middleName = value[property];
+                        continue;
+                    } else if (this.isNullOrWhitespace(cipher.identity.lastName) && lowerProp.includes('last name')) {
+                        cipher.identity.lastName = value[property];
+                        continue;
+                    } else if (this.isNullOrWhitespace(cipher.identity.username) && lowerProp.includes('username')) {
+                        cipher.identity.username = value[property];
+                        continue;
+                    } else if (this.isNullOrWhitespace(cipher.identity.company) && lowerProp.includes('company')) {
+                        cipher.identity.company = value[property];
+                        continue;
+                    } else if (this.isNullOrWhitespace(cipher.identity.phone) && lowerProp.includes('default phone')) {
+                        cipher.identity.phone = value[property];
+                        continue;
+                    } else if (this.isNullOrWhitespace(cipher.identity.email) && lowerProp.includes('email')) {
+                        cipher.identity.email = value[property];
+                        continue;
+                    }
                 }
 
                 if (IgnoredProperties.indexOf(lowerProp) === -1 && !lowerProp.startsWith('section:') &&
@@ -81,6 +135,11 @@ export class OnePasswordWinCsvImporter extends BaseImporter implements Importer
                     if (altUsername == null && lowerProp === 'email') {
                         altUsername = value[property];
                     }
+                    else if (lowerProp === 'created date' || lowerProp === 'modified date') {
+                        const readableDate = new Date(parseInt(value[property], 10) * 1000).toUTCString();
+                        this.processKvp(cipher, '1Password ' + property, readableDate);
+                        continue;
+                    }
                     this.processKvp(cipher, property, value[property]);
                 }
             }
@@ -100,6 +159,10 @@ export class OnePasswordWinCsvImporter extends BaseImporter implements Importer
     }
 
     private getProp(obj: any, name: string): any {
-        return obj[name] || obj[name.toUpperCase()];
+        const lowerObj = Object.entries(obj).reduce((agg: any, entry: [string, any]) => {
+            agg[entry[0].toLowerCase()] = entry[1];
+            return agg;
+        }, {});
+        return lowerObj[name.toLowerCase()];
     }
 }