Move all functions to scripts folder

package.json

@@ -2,9 +2,9 @@
   "name": "iptv",
   "scripts": {
     "test": "node test/index.js",
-    "generate": "node helpers/generate.js",
+    "generate": "node scripts/generate.js",
-    "format": "node helpers/format.js",
+    "format": "node scripts/format.js",
-    "update-readme": "node helpers/update-readme.js"
+    "update-readme": "node scripts/update-readme.js"
   },
   "author": "Arhey <free.arhey@gmail.com>",
   "private": true,

scripts/format.js

@@ -1,6 +1,4 @@
-const util = require('./util')
+const helper = require('./helper')
-const escapeStringRegexp = require('escape-string-regexp')
-const ISO6391 = require('iso-639-1')
 
 const debug = false
 const verbose = false
@@ -19,46 +17,46 @@ let unsorted = {}
 async function main() {
 
   console.log(`Parsing 'index.m3u'...`)
-  const playlist = util.parsePlaylist('index.m3u')
+  const playlist = helper.parsePlaylist('index.m3u')
   const countries = playlist.items
 
   if(debug) {
     console.log('Debug mode is turn on')
   }
 
-  const unsortedPlaylist = util.parsePlaylist('channels/unsorted.m3u')
+  const unsortedPlaylist = helper.parsePlaylist('channels/unsorted.m3u')
   for(const item of unsortedPlaylist.items) {
-    unsorted[item.url] = util.createChannel(item)
+    unsorted[item.url] = helper.createChannel(item)
   }
 
   for(let country of countries) {
 
-    if (util.skipPlaylist(country.url)) {
+    if (helper.skipPlaylist(country.url)) {
       continue
     }
     
     if(verbose) {
       console.log(`Clear cache...`)
     }
-    util.clearCache()
+    helper.clearCache()
 
     console.log(`Parsing '${country.url}'...`)
-    const playlist = util.parsePlaylist(country.url)
+    const playlist = helper.parsePlaylist(country.url)
 
     if(verbose) {
       console.log(`Creating channels list...`)
     }
     let channels = []
     for(let item of playlist.items) {
-      let channel = util.createChannel(item)
+      let channel = helper.createChannel(item)
 
-      if(util.checkCache(channel.url)) {
+      if(helper.checkCache(channel.url)) {
         stats.duplicates++
-      } else if(!util.validateUrl(channel.url)) {
+      } else if(!helper.validateUrl(channel.url)) {
         stats.unvalid++
       } else {
         channels.push(channel)
-        util.addToCache(channel.url)
+        helper.addToCache(channel.url)
       }
 
       if(unsorted[channel.url]) {
@@ -75,7 +73,7 @@ async function main() {
     if(epgUrl && !buffer[epgUrl] && parseEpg) {
       try {
         console.log(`Loading '${epgUrl}'...`)
-        const epg = await util.loadEPG(epgUrl)
+        const epg = await helper.loadEPG(epgUrl)
         console.log(`Adding '${epgUrl}' to buffer...`)
         buffer[epgUrl] = epg
       } catch(e) {
@@ -90,7 +88,7 @@ async function main() {
           let c = buffer[epgUrl].channels[channelId]
           for(let epgName of c.name) {
             if(epgName.value) {
-              let escaped = escapeStringRegexp(epgName.value)
+              let escaped = helper.escapeStringRegexp(epgName.value)
               channelTitle = channel.title.replace(/(fhd|hd|sd|高清)$/i, '').trim()
               let regexp = new RegExp(`^${escaped}$`, 'i')
               if(regexp.test(channelTitle)) {
@@ -123,7 +121,7 @@ async function main() {
         }
 
         if(!channel.language && c.name.length && c.name[0].lang) {
-          let language = ISO6391.getName(c.name[0].lang)
+          let language = helper.getISO6391Name(c.name[0].lang)
           channel.language = language
           updated = true
           if(verbose) {
@@ -149,13 +147,13 @@ async function main() {
     if(verbose) {
       console.log(`Sorting channels...`)
     }
-    channels = util.sortByTitleAndUrl(channels)
+    channels = helper.sortByTitleAndUrl(channels)
 
     if(!debug) {
       console.log(`Updating '${country.url}'...`)
-      util.createFile(country.url, playlist.getHeader())
+      helper.createFile(country.url, playlist.getHeader())
       channels.forEach(channel => {
-        util.appendToFile(country.url, channel.toString())
+        helper.appendToFile(country.url, channel.toString())
       })
     }
 
@@ -164,9 +162,9 @@ async function main() {
 
   if(!debug & stats.removed > 0) {
     console.log(`Updating 'channels/unsorted.m3u'...`)
-    util.createFile('channels/unsorted.m3u', playlist.getHeader())
+    helper.createFile('channels/unsorted.m3u', playlist.getHeader())
     Object.values(unsorted).forEach(channel => {
-      util.appendToFile('channels/unsorted.m3u', channel.toString())
+      helper.appendToFile('channels/unsorted.m3u', channel.toString())
     })
   }
 

scripts/generate.js

@@ -1,5 +1,4 @@
-const util = require('./util')
+const helper = require('./helper')
-const ISO6391 = require('iso-639-1')
 
 let list = {
   all: [],
@@ -29,19 +28,19 @@ function main() {
 }
 
 function parseIndex() {
-  const root = util.parsePlaylist('index.m3u')
+  const root = helper.parsePlaylist('index.m3u')
 
   let countries = {}
   let languages = {}
   let categories = {}
 
   for(let rootItem of root.items) {
-    const playlist = util.parsePlaylist(rootItem.url)
+    const playlist = helper.parsePlaylist(rootItem.url)
-    const countryCode = util.getBasename(rootItem.url).toLowerCase()
+    const countryCode = helper.getBasename(rootItem.url).toLowerCase()
     const countryName = rootItem.name
 
     for(let item of playlist.items) {
-      const channel = util.createChannel(item)
+      const channel = helper.createChannel(item)
       channel.countryCode = countryCode
       channel.countryName = countryName
 
@@ -55,7 +54,7 @@ function parseIndex() {
       countries[countryCode].push(channel)
 
       // language
-      const languageCode = ISO6391.getCode(channel.language) || 'undefined'
+      const languageCode = helper.getISO6391Code(channel.language) || 'undefined'
       if(!languages[languageCode]) {
         languages[languageCode] = []
       }
@@ -77,19 +76,19 @@ function parseIndex() {
 
 function generateCountryIndex() {
   const filename = `index.country.m3u`
-  util.createFile(filename, '#EXTM3U\n')
+  helper.createFile(filename, '#EXTM3U\n')
 
   for(let channel of list.all) {
     const group = channel.group
     channel.group = channel.countryName
-    util.appendToFile(filename, channel.toString())
+    helper.appendToFile(filename, channel.toString())
     channel.group = group
   }
 }
 
 function generateLanguageIndex() {
   const filename = `index.language.m3u`
-  util.createFile(filename, '#EXTM3U\n')
+  helper.createFile(filename, '#EXTM3U\n')
 
   const channels = list.all.sort((a, b) => {
     if(a.language < b.language) { return -1 }
@@ -100,14 +99,14 @@ function generateLanguageIndex() {
   for(let channel of channels) {
     const group = channel.group
     channel.group = channel.language
-    util.appendToFile(filename, channel.toString())
+    helper.appendToFile(filename, channel.toString())
     channel.group = group
   }
 }
 
 function generateContentIndex() {
   const filename = `index.content.m3u`
-  util.createFile(filename, '#EXTM3U\n')
+  helper.createFile(filename, '#EXTM3U\n')
 
   const channels = list.all.sort((a, b) => {
     if(a.group < b.group) { return -1 }
@@ -116,13 +115,13 @@ function generateContentIndex() {
   })
 
   for(let channel of channels) {
-    util.appendToFile(filename, channel.toString())
+    helper.appendToFile(filename, channel.toString())
   }
 }
 
 function generateFullIndex() {
   const filename = `index.full.m3u`
-  util.createFile(filename, '#EXTM3U\n')
+  helper.createFile(filename, '#EXTM3U\n')
 
   const channels = list.all.sort((a, b) => {
     if(a.countryName < b.countryName) { return -1 }
@@ -135,7 +134,7 @@ function generateFullIndex() {
   for(let channel of channels) {
     const group = channel.group
     channel.group = [ channel.countryName, channel.group ].filter(i => i).join(';')
-    util.appendToFile(filename, channel.toString())
+    helper.appendToFile(filename, channel.toString())
     channel.group = group
   }
 }
@@ -144,9 +143,9 @@ function generateCategories() {
   for(let cid in list.categories) {
     let category = list.categories[cid]
     const filename = `categories/${cid}.m3u`
-    util.createFile(filename, '#EXTM3U\n')
+    helper.createFile(filename, '#EXTM3U\n')
     for(let channel of category) {
-      util.appendToFile(filename, channel.toString())
+      helper.appendToFile(filename, channel.toString())
     }
   }
 }
@@ -155,9 +154,9 @@ function generateLanguages() {
   for(let lid in list.languages) {
     let language = list.languages[lid]
     const filename = `languages/${lid}.m3u`
-    util.createFile(filename, '#EXTM3U\n')
+    helper.createFile(filename, '#EXTM3U\n')
     for(let channel of language) {
-      util.appendToFile(filename, channel.toString())
+      helper.appendToFile(filename, channel.toString())
     }
   }
 }

scripts/helper.js

@@ -6,88 +6,48 @@ const zlib = require("zlib")
 const epgParser = require('epg-parser')
 const urlParser = require('url')
 const ISO6391 = require('iso-639-1')
-
+const escapeStringRegexp = require('escape-string-regexp')
-const supportedCategories = [ 'Auto','Business', 'Classic','Comedy','Documentary','Education','Entertainment', 'Family','Fashion','Food', 'General', 'Health', 'History', 'Hobby', 'Kids', 'Legislative','Lifestyle','Local', 'Movies', 'Music', 'News', 'Quiz', 'Religious','Sci-Fi', 'Shop', 'Sport', 'Travel', 'Weather', 'XXX' ]
+const markdownInclude = require('markdown-include')
-
-const blacklist = [
-  '80.80.160.168', // repeats on a loop
-  '63.237.48.3', // not a live stream
-  '189.216.247.113', // not working streams
-]
 
 let cache = {}
+let helper = {}
 
-class Playlist {
+helper.compileMarkdown = function(filepath) {
-  constructor(data) {
+  return markdownInclude.compileFiles(path.resolve(__dirname, filepath))
-    this.header = data.header
-    this.items = data.items
 }
 
-  getHeader() {
+helper.escapeStringRegexp = function(scring) {
-    let parts = ['#EXTM3U']
+  return escapeStringRegexp(string)
-    for(let key in this.header.attrs) {
-      let value = this.header.attrs[key]
-      if(value) {
-        parts.push(`${key}="${value}"`)
-      }
 }
 
-    return `${parts.join(' ')}\n`
+helper.getISO6391Name = function(code) {
-  }
+  return ISO6391.getName(code)
 }
 
-class Channel {
+helper.getISO6391Code = function(name) {
-  constructor(data) {
+  return ISO6391.getCode(name)
-    this.id = data.tvg.id
-    this.name = data.tvg.name
-    this.language = this._filterLanguage(data.tvg.language)
-    this.logo = data.tvg.logo
-    this.group = this._filterGroup(data.group.title)
-    this.url = data.url
-    this.title = data.name
 }
 
-  _filterGroup(groupTitle) {
+helper.parsePlaylist = function(filename) {
-    if(!groupTitle) return ''
-      
-    const groupIndex = supportedCategories.map(g => g.toLowerCase()).indexOf(groupTitle.toLowerCase())
-
-    if(groupIndex === -1) {
-      groupTitle = ''
-    } else {
-      groupTitle = supportedCategories[groupIndex]
-    }
-
-    return groupTitle
-  }
-
-  _filterLanguage(languageName) {
-    if(ISO6391.getCode(languageName) !== '') {
-      return languageName
-    }
-
-    return ''
-  }
-
-  toString() {
-    const info = `-1 tvg-id="${this.id}" tvg-name="${this.name}" tvg-language="${this.language}" tvg-logo="${this.logo}" group-title="${this.group}",${this.title}`
-
-    return '#EXTINF:' + info + '\n' + this.url + '\n'
-  }
-}
-
-function parsePlaylist(filename) {
   const content = readFile(filename)
   const result = parser.parse(content)
 
   return new Playlist(result)
 }
 
-function createChannel(data) {
+helper.createChannel = function(data) {
-  return new Channel(data)
+  return new Channel({
+    id: data.tvg.id,
+    name: data.tvg.name,
+    language: data.tvg.language,
+    logo: data.tvg.logo,
+    group: data.group.title,
+    url: data.url,
+    title: data.name
+  })
 }
 
-async function loadEPG(url) {
+helper.loadEPG = async function(url) {
   const content = await getEPGFile(url)
   const result = epgParser.parse(content)
   const channels = {}
@@ -101,7 +61,7 @@ async function loadEPG(url) {
   })
 }
 
-function getEPGFile(url) {
+helper.getEPGFile = function(url) {
   return new Promise((resolve, reject) => {
     var buffer = []
     axios({
@@ -131,7 +91,8 @@ function getEPGFile(url) {
   })
 }
 
-function byTitleAndUrl(a, b) {
+helper.sortByTitleAndUrl = function(arr) {
+  return arr.sort(function(a, b) {
     var titleA = a.title.toLowerCase()
     var titleB = b.title.toLowerCase()
     var urlA = a.url.toLowerCase()
@@ -144,45 +105,42 @@ function byTitleAndUrl(a, b) {
     if(urlA > urlB) return 1
 
     return 0
+  })
 }
 
-function sortByTitleAndUrl(arr) {
+helper.readFile = function(filename) {
-  return arr.sort(byTitleAndUrl)
-}
-
-function readFile(filename) {
   return fs.readFileSync(path.resolve(__dirname) + `/../${filename}`, { encoding: "utf8" })
 }
 
-function appendToFile(filename, data) {
+helper.appendToFile = function(filename, data) {
   fs.appendFileSync(path.resolve(__dirname) + '/../' + filename, data)
 }
 
-function createFile(filename, data) {
+helper.createFile = function(filename, data) {
   fs.writeFileSync(path.resolve(__dirname) + '/../' + filename, data)
 }
 
-function getBasename(filename) {
+helper.getBasename = function(filename) {
   return path.basename(filename, path.extname(filename))
 }
 
-function addToCache(url) {
+helper.addToCache = function(url) {
   let id = getUrlPath(url)
 
   cache[id] = true
 }
 
-function checkCache(url) {
+helper.checkCache = function(url) {
   let id = getUrlPath(url)
 
   return cache.hasOwnProperty(id)
 }
 
-function clearCache() {
+helper.clearCache = function() {
   cache = {}
 }
 
-function getUrlPath(u) {
+helper.getUrlPath = function(u) {
   let parsed = urlParser.parse(u)
   let searchQuery = parsed.search || ''
   let path = parsed.host + parsed.pathname + searchQuery
@@ -190,14 +148,19 @@ function getUrlPath(u) {
   return path.toLowerCase()
 }
 
-function validateUrl(channelUrl) {
+helper.validateUrl = function(channelUrl) {
   const url = new URL(channelUrl)
   const host = url.hostname
+  const blacklist = [
+    '80.80.160.168', // repeats on a loop
+    '63.237.48.3', // not a live stream
+    '189.216.247.113', // not working streams
+  ]
 
   return blacklist.indexOf(host) === -1
 }
 
-function skipPlaylist(filename) {
+helper.skipPlaylist = function(filename) {
   let testCountry = process.env.npm_config_country
   let excludeList = process.env.npm_config_exclude
   let excludeCountries = excludeList ? excludeList.split(',') : []
@@ -211,7 +174,7 @@ function skipPlaylist(filename) {
   return false
 }
 
-function generateTable(data, options) {
+helper.generateTable = function(data, options) {
   let output = '<table>'
 
   output += '<thead><tr>'
@@ -240,20 +203,64 @@ function generateTable(data, options) {
   return output
 }
 
-module.exports = {
+class Playlist {
-  parsePlaylist,
+  constructor(data) {
-  sortByTitleAndUrl,
+    this.header = data.header
-  appendToFile,
+    this.items = data.items
-  createFile,
-  readFile,
-  loadEPG,
-  createChannel,
-  getBasename,
-  addToCache,
-  checkCache,
-  clearCache,
-  validateUrl,
-  skipPlaylist,
-  supportedCategories,
-  generateTable
   }
+
+  getHeader() {
+    let parts = ['#EXTM3U']
+    for(let key in this.header.attrs) {
+      let value = this.header.attrs[key]
+      if(value) {
+        parts.push(`${key}="${value}"`)
+      }
+    }
+
+    return `${parts.join(' ')}\n`
+  }
+}
+
+class Channel {
+  constructor(data) {
+    this.id = data.id
+    this.name = data.name
+    this.language = this._filterLanguage(data.language)
+    this.logo = data.logo
+    this.group = this._filterGroup(data.group)
+    this.url = data.url
+    this.title = data.title
+  }
+
+  _filterGroup(groupTitle) {
+    if(!groupTitle) return ''
+      
+    const supportedCategories = [ 'Auto','Business', 'Classic','Comedy','Documentary','Education','Entertainment', 'Family','Fashion','Food', 'General', 'Health', 'History', 'Hobby', 'Kids', 'Legislative','Lifestyle','Local', 'Movies', 'Music', 'News', 'Quiz', 'Religious','Sci-Fi', 'Shop', 'Sport', 'Travel', 'Weather', 'XXX' ]
+    const groupIndex = supportedCategories.map(g => g.toLowerCase()).indexOf(groupTitle.toLowerCase())
+
+    if(groupIndex === -1) {
+      groupTitle = ''
+    } else {
+      groupTitle = supportedCategories[groupIndex]
+    }
+
+    return groupTitle
+  }
+
+  _filterLanguage(languageName) {
+    if(ISO6391.getCode(languageName) !== '') {
+      return languageName
+    }
+
+    return ''
+  }
+
+  toString() {
+    const info = `-1 tvg-id="${this.id}" tvg-name="${this.name}" tvg-language="${this.language}" tvg-logo="${this.logo}" group-title="${this.group}",${this.title}`
+
+    return '#EXTINF:' + info + '\n' + this.url + '\n'
+  }
+}
+
+module.exports = helper

scripts/update-readme.js

@@ -1,7 +1,4 @@
-const util = require('./util')
+const helper = require('./helper')
-const ISO6391 = require('iso-639-1')
-const markdownInclude = require('markdown-include')
-const path = require('path')
 
 let output = {
   countries: [],
@@ -24,13 +21,13 @@ function main() {
 }
 
 function parseIndex() {
-  const root = util.parsePlaylist('index.m3u')
+  const root = helper.parsePlaylist('index.m3u')
 
   let languages = {}
   let categories = {}
   for(let rootItem of root.items) {
-    const playlist = util.parsePlaylist(rootItem.url)
+    const playlist = helper.parsePlaylist(rootItem.url)
-    const countryCode = util.getBasename(rootItem.url).toUpperCase()
+    const countryCode = helper.getBasename(rootItem.url).toUpperCase()
     const epg = playlist.header.attrs['x-tvg-url'] ? `<code>${playlist.header.attrs['x-tvg-url']}</code>` : ''
 
     // country
@@ -45,7 +42,7 @@ function parseIndex() {
       
       // language
       const languageName = item.tvg.language || 'Undefined'
-      const languageCode = ISO6391.getCode(languageName) || 'undefined'
+      const languageCode = helper.getISO6391Code(languageName) || 'undefined'
       if(languages[languageCode]) { 
         languages[languageCode].channels++
       } else {
@@ -57,7 +54,7 @@ function parseIndex() {
       }
 
       // category
-      const categoryName = util.supportedCategories.find(c => c === item.group.title) || 'Other'
+      const categoryName = item.group.title || 'Other'
       const categoryCode = categoryName.toLowerCase()
       if(categories[categoryCode]) {
         categories[categoryCode].channels++
@@ -76,7 +73,7 @@ function parseIndex() {
 }
 
 function generateCountriesTable() {
-  const table = util.generateTable(output.countries, {
+  const table = helper.generateTable(output.countries, {
     columns: [
       { name: 'Country', align: 'left' },
       { name: 'Channels', align: 'right' },
@@ -85,7 +82,7 @@ function generateCountriesTable() {
     ]
   })
 
-  util.createFile('./.readme/_countries.md', table)
+  helper.createFile('./.readme/_countries.md', table)
 }
 
 function generateLanguagesTable() {
@@ -97,7 +94,7 @@ function generateLanguagesTable() {
     return 0
   })
 
-  const table = util.generateTable(output.languages, {
+  const table = helper.generateTable(output.languages, {
     columns: [
       { name: 'Language', align: 'left' },
       { name: 'Channels', align: 'right' },
@@ -105,7 +102,7 @@ function generateLanguagesTable() {
     ]
   })
 
-  util.createFile('./.readme/_languages.md', table)
+  helper.createFile('./.readme/_languages.md', table)
 }
 
 function generateCategoriesTable() {
@@ -117,7 +114,7 @@ function generateCategoriesTable() {
     return 0
   })
 
-  const table = util.generateTable(output.categories, {
+  const table = helper.generateTable(output.categories, {
     columns: [
       { name: 'Category', align: 'left' },
       { name: 'Channels', align: 'right' },
@@ -125,11 +122,11 @@ function generateCategoriesTable() {
     ]
   })
 
-  util.createFile('./.readme/_categories.md', table)
+  helper.createFile('./.readme/_categories.md', table)
 }
 
 function generateReadme() {
-  markdownInclude.compileFiles(path.resolve(__dirname, '../.readme/config.json'))
+  helper.compileMarkdown('../.readme/config.json')
 }
 
 main()

test/index.js

@@ -1,6 +1,6 @@
 process.env['NODE_TLS_REJECT_UNAUTHORIZED'] = 0
 
-const util = require('../helpers/util')
+const helper = require('../scripts/helper')
 const ffmpeg = require('fluent-ffmpeg')
 
 const verbose = process.env.npm_config_debug || false
@@ -19,19 +19,19 @@ async function test() {
 
   stats.tests++
 
-  const playlist = util.parsePlaylist('index.m3u')
+  const playlist = helper.parsePlaylist('index.m3u')
   
   const countries = playlist.items
 
   for(let country of countries) {
 
-    if (util.skipPlaylist(country.url)) {
+    if (helper.skipPlaylist(country.url)) {
 	    continue
     }
 
     console.log(`Checking '${country.url}'...`)
 
-    const playlist = util.parsePlaylist(country.url)
+    const playlist = helper.parsePlaylist(country.url)
 
     for(let item of playlist.items) {
 
@@ -90,7 +90,7 @@ test()
 function writeToLog(country, msg, url) {
   var now = new Date()
   var line = `${country}: ${msg} '${url}'`
-  util.appendToFile(errorLog, now.toISOString() + ' ' + line + '\n')
+  helper.appendToFile(errorLog, now.toISOString() + ' ' + line + '\n')
   console.log(`${msg} '${url}'`)
 }