Merge pull request #3823 from iptv-org/patch-2

Update scripts and GitHub actions

.github/workflows/auto-update.yml

@@ -4,103 +4,382 @@ on:
   schedule:
     - cron: '0 0 * * *'
 jobs:
-  remove-duplicates:
+  create-branch:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
         uses: actions/checkout@v2
-      - name: Install Dependencies
-        run: npm install
-      - name: Remove Duplicates
-        run: node scripts/remove-duplicates.js
-      - name: Upload Artifact
-        uses: actions/upload-artifact@v2
         with:
-          name: channels
+          ref: ${{ github.ref }}
-          path: channels/
+      - name: Create Branch
-  filter:
+        uses: peterjgrainger/action-create-branch@v2.0.1
-    runs-on: ubuntu-latest
+        env:
-    needs: remove-duplicates
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v2
-      - name: Download Artifacts
-        uses: actions/download-artifact@v2
-      - name: Install Dependencies
-        run: npm install
-      - name: Filter Playlists
-        run: node scripts/filter.js
-      - name: Upload Artifact
-        uses: actions/upload-artifact@v2
         with:
-          name: channels
+          branch: 'bot/auto-update'
-          path: channels/
   format:
     runs-on: ubuntu-latest
-    needs: filter
+    needs: create-branch
     steps:
       - name: Checkout
         uses: actions/checkout@v2
-      - name: Download Artifacts
+        with:
-        uses: actions/download-artifact@v2
+          ref: bot/auto-update
       - name: Install Dependencies
         run: npm install
       - name: Format Playlists
         run: node scripts/format.js
-      - name: Upload Artifact
+      - name: Commit Changes
-        uses: actions/upload-artifact@v2
+        uses: stefanzweifel/git-auto-commit-action@v4
         with:
-          name: channels
+          commit_message: '[Bot] Formate playlists'
-          path: channels/
+          commit_user_name: iptv-bot
-  generate:
+          commit_user_email: 84861620+iptv-bot[bot]@users.noreply.github.com
+          commit_author: 'iptv-bot[bot] <84861620+iptv-bot[bot]@users.noreply.github.com>'
+          branch: bot/auto-update
+          file_pattern: channels/*
+  remove-duplicates:
     runs-on: ubuntu-latest
     needs: format
     steps:
       - name: Checkout
         uses: actions/checkout@v2
+        with:
+          ref: bot/auto-update
+      - name: Install Dependencies
+        run: npm install
+      - name: Remove Duplicates
+        run: node scripts/remove-duplicates.js
+      - name: Commit Changes
+        uses: stefanzweifel/git-auto-commit-action@v4
+        with:
+          commit_message: '[Bot] Remove duplicates'
+          commit_user_name: iptv-bot
+          commit_user_email: 84861620+iptv-bot[bot]@users.noreply.github.com
+          commit_author: 'iptv-bot[bot] <84861620+iptv-bot[bot]@users.noreply.github.com>'
+          branch: bot/auto-update
+          file_pattern: channels/*
+  sort:
+    runs-on: ubuntu-latest
+    needs: remove-duplicates
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          ref: bot/auto-update
+      - name: Install Dependencies
+        run: npm install
+      - name: Sort Channels
+        run: node scripts/sort.js
+      - name: Commit Changes
+        uses: stefanzweifel/git-auto-commit-action@v4
+        with:
+          commit_message: '[Bot] Sort channels'
+          commit_user_name: iptv-bot
+          commit_user_email: 84861620+iptv-bot[bot]@users.noreply.github.com
+          commit_author: 'iptv-bot[bot] <84861620+iptv-bot[bot]@users.noreply.github.com>'
+          branch: bot/auto-update
+          file_pattern: channels/*
+  filter:
+    runs-on: ubuntu-latest
+    needs: sort
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          ref: bot/auto-update
+      - name: Install Dependencies
+        run: npm install
+      - name: Filter Playlists
+        run: node scripts/filter.js
+      - name: Commit Changes
+        uses: stefanzweifel/git-auto-commit-action@v4
+        with:
+          commit_message: '[Bot] Filter channels'
+          commit_user_name: iptv-bot
+          commit_user_email: 84861620+iptv-bot[bot]@users.noreply.github.com
+          commit_author: 'iptv-bot[bot] <84861620+iptv-bot[bot]@users.noreply.github.com>'
+          branch: bot/auto-update
+          file_pattern: channels/*
+  detect-resolution:
+    runs-on: ubuntu-latest
+    needs: filter
+    continue-on-error: true
+    strategy:
+      fail-fast: false
+      matrix:
+        country:
+          [
+            ad,
+            ae,
+            af,
+            ag,
+            al,
+            am,
+            an,
+            ao,
+            ar,
+            at,
+            au,
+            aw,
+            az,
+            ba,
+            bb,
+            bd,
+            be,
+            bf,
+            bg,
+            bh,
+            bn,
+            bo,
+            br,
+            bs,
+            by,
+            ca,
+            cd,
+            cg,
+            ch,
+            ci,
+            cl,
+            cm,
+            cn,
+            co,
+            cr,
+            cu,
+            cw,
+            cy,
+            cz,
+            de,
+            dk,
+            do,
+            dz,
+            ec,
+            ee,
+            eg,
+            es,
+            et,
+            fi,
+            fj,
+            fo,
+            fr,
+            ge,
+            gh,
+            gm,
+            gn,
+            gp,
+            gq,
+            gr,
+            gt,
+            hk,
+            hn,
+            hr,
+            ht,
+            hu,
+            id,
+            ie,
+            il,
+            in,
+            iq,
+            ir,
+            is,
+            it,
+            jm,
+            jo,
+            jp,
+            ke,
+            kg,
+            kh,
+            kp,
+            kr,
+            kw,
+            kz,
+            la,
+            lb,
+            li,
+            lk,
+            lt,
+            lu,
+            lv,
+            ly,
+            ma,
+            mc,
+            md,
+            me,
+            mk,
+            ml,
+            mm,
+            mn,
+            mo,
+            mt,
+            mv,
+            mx,
+            my,
+            mz,
+            ne,
+            ng,
+            ni,
+            nl,
+            no,
+            np,
+            nz,
+            om,
+            pa,
+            pe,
+            ph,
+            pk,
+            pl,
+            pr,
+            ps,
+            pt,
+            py,
+            qa,
+            ro,
+            rs,
+            ru,
+            rw,
+            sa,
+            sd,
+            se,
+            sg,
+            si,
+            sk,
+            sl,
+            sm,
+            sn,
+            so,
+            sv,
+            sy,
+            th,
+            tj,
+            tm,
+            tn,
+            tr,
+            tt,
+            tw,
+            tz,
+            ua,
+            ug,
+            uk,
+            us,
+            uy,
+            uz,
+            va,
+            ve,
+            vi,
+            vn,
+            xk,
+            ye,
+            zm
+          ]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          ref: bot/auto-update
+      - name: Install Dependencies
+        run: npm install
+      - name: Detect Resolution
+        run: node scripts/detect-resolution.js --country=${{ matrix.country }}
+      - name: Upload Artifact
+        uses: actions/upload-artifact@v2
+        with:
+          name: channels
+          path: channels/${{ matrix.country }}.m3u
+  commit-changes:
+    runs-on: ubuntu-latest
+    needs: detect-resolution
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          ref: bot/auto-update
       - name: Download Artifacts
         uses: actions/download-artifact@v2
+        with:
+          name: channels
+      - name: Commit Changes
+        uses: stefanzweifel/git-auto-commit-action@v4
+        with:
+          commit_message: '[Bot] Detect resolution'
+          commit_user_name: iptv-bot
+          commit_user_email: 84861620+iptv-bot[bot]@users.noreply.github.com
+          commit_author: 'iptv-bot[bot] <84861620+iptv-bot[bot]@users.noreply.github.com>'
+          branch: bot/auto-update
+          file_pattern: channels/*
+  generate:
+    runs-on: ubuntu-latest
+    needs: commit-changes
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          ref: bot/auto-update
       - name: Install Dependencies
         run: npm install
       - name: Generate Playlists
         run: node scripts/generate.js
+      - name: Upload Artifact
+        uses: actions/upload-artifact@v2
+        with:
+          name: gh-pages
+          path: .gh-pages/
+  deploy:
+    if: ${{ github.ref == 'refs/heads/master' }}
+    runs-on: ubuntu-latest
+    needs: generate
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          ref: bot/auto-update
+      - name: Download Artifacts
+        uses: actions/download-artifact@v2
+        with:
+          name: gh-pages
+      - name: Generate Token
+        uses: tibdex/github-app-token@v1
+        id: generate-token
+        with:
+          app_id: ${{ secrets.APP_ID }}
+          private_key: ${{ secrets.APP_PRIVATE_KEY }}
       - name: Deploy to GitHub Pages
         uses: JamesIves/github-pages-deploy-action@4.1.1
         with:
           branch: gh-pages
-          folder: .gh-pages
+          folder: gh-pages
+          token: ${{ steps.generate-token.outputs.token }}
+          git-config-name: iptv-bot
+          git-config-email: 84861620+iptv-bot[bot]@users.noreply.github.com
+          commit-message: '[Bot] Deploy to GitHub Pages'
   update-readme:
     runs-on: ubuntu-latest
     needs: generate
     steps:
       - name: Checkout
         uses: actions/checkout@v2
-      - name: Download Artifacts
+        with:
-        uses: actions/download-artifact@v2
+          ref: bot/auto-update
       - name: Install Dependencies
         run: npm install
       - name: Update README.md
         run: node scripts/update-readme.js
-      - name: Upload Artifact
+      - name: Commit Changes
-        uses: actions/upload-artifact@v2
+        uses: stefanzweifel/git-auto-commit-action@v4
         with:
-          name: README.md
+          commit_message: '[Bot] Update README.md'
-          path: README.md
+          commit_user_name: iptv-bot
+          commit_user_email: 84861620+iptv-bot[bot]@users.noreply.github.com
+          commit_author: 'iptv-bot[bot] <84861620+iptv-bot[bot]@users.noreply.github.com>'
+          branch: bot/auto-update
+          file_pattern: README.md
   pull-request:
     needs: update-readme
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
         uses: actions/checkout@v2
-      - name: Download /channels
-        uses: actions/download-artifact@v2
         with:
-          name: channels
+          ref: bot/auto-update
-          path: channels/
-      - name: Download README.md
-        uses: actions/download-artifact@v2
-        with:
-          name: README.md
       - name: Generate Token
         uses: tibdex/github-app-token@v1
         id: generate-token
@@ -109,26 +388,25 @@ jobs:
           private_key: ${{ secrets.APP_PRIVATE_KEY }}
       - name: Create Pull Request
         id: pr
-        uses: peter-evans/create-pull-request@v3
+        uses: repo-sync/pull-request@v2
         with:
-          title: '[Bot] Update playlists'
+          source_branch: 'bot/auto-update'
-          body: |
+          destination_branch: 'master'
-            This pull request is created automatically by `auto-update` action.
+          pr_title: '[Bot] Update playlists'
-          commit-message: '[Bot] Update playlists'
+          pr_body: |
-          committer: GitHub <noreply@github.com>
+            This pull request is created by [auto-update][1] workflow.
-          branch: bot/auto-update
+
-          delete-branch: true
+            [1]: https://github.com/iptv-org/iptv/actions/runs/${{ github.run_id }}
-          token: ${{ steps.generate-token.outputs.token }}
+          github_token: ${{ steps.generate-token.outputs.token }}
       - name: Enable Pull Request Automerge
-        if: steps.pr.outputs.pull-request-operation == 'created'
         uses: peter-evans/enable-pull-request-automerge@v1
         with:
           token: ${{ secrets.PAT }}
-          pull-request-number: ${{ steps.pr.outputs.pull-request-number }}
+          pull-request-number: ${{ steps.pr.outputs.pr_number }}
           merge-method: squash
       - name: Approve Pull Request
-        if: steps.pr.outputs.pull-request-operation == 'created'
+        if: github.ref == 'refs/heads/master'
         uses: juliangruber/approve-pull-request-action@v1
         with:
           github-token: ${{ secrets.PAT }}
-          number: ${{ steps.pr.outputs.pull-request-number }}
+          number: ${{ steps.pr.outputs.pr_number }}

CONTRIBUTING.md

@@ -167,18 +167,15 @@ http://example.com/stream.m3u8
   - ...
   - `unsorted.m3u`: playlist with channels not yet sorted.
 - `scripts/`
-  - `blacklist.json`: list of channels banned for addition to the repository.
+  - `helpers/`: helper scripts used in GitHub Actions.
-  - `categories.json`: list of supported categories.
   - `clean.js`: used in GitHub Action to check all links and remove broken ones.
-  - `db.js`: contains functions for retrieving and managing the channel list.
+  - `detect-resolution.js`: used in GitHub Action to detect resolution of the streams.
   - `filter.js`: used within GitHub Action to remove blacklisted channels from playlists.
-  - `format.js`: used within GitHub Action to format channel descriptions and sort playlists.
+  - `format.js`: used within GitHub Action to format channel descriptions.
   - `generate.js`: used within GitHub Action to generate all additional playlists.
-  - `parser.js`: contains functions for parsing playlists.
-  - `regions.json`: list of supported region codes.
   - `remove-duplicates.js`: used in GitHub Action to remove duplicates from the playlist.
+  - `sort.js`: used within GitHub Action to sort channels by name.
   - `update-readme.js`: used within GitHub Action to update the `README.md` file.
-  - `utils.js`: contains functions that are used in other scripts.
 - `CONTRIBUTING.md`: file you are currently reading.
 - `index.m3u`: main playlist that contains links to all playlists in the `channels/` folder.
 - `README.md`: project description generated from the contents of the `.readme/` folder.

scripts/clean.js

@@ -1,14 +1,14 @@
 const { program } = require('commander')
-const parser = require('./parser')
-const utils = require('./utils')
-const axios = require('axios')
 const ProgressBar = require('progress')
+const axios = require('axios')
 const https = require('https')
 const chalk = require('chalk')
+const parser = require('./helpers/parser')
+const utils = require('./helpers/utils')
+const log = require('./helpers/log')
 
 program
   .usage('[OPTIONS]...')
-  .option('-d, --debug', 'Debug mode')
   .option('-c, --country <country>', 'Comma-separated list of country codes', '')
   .option('-e, --exclude <exclude>', 'Comma-separated list of country codes to be excluded', '')
   .option('--delay <delay>', 'Delay between parser requests', 1000)
@@ -16,8 +16,8 @@ program
   .parse(process.argv)
 
 const config = program.opts()
-
 const offlineStatusCodes = [404, 410, 500, 501]
+const ignore = ['Geo-blocked', 'Not 24/7']
 const instance = axios.create({
   timeout: config.timeout,
   maxContentLength: 200000,
@@ -29,60 +29,39 @@ const instance = axios.create({
   }
 })
 
-const ignore = ['Geo-blocked', 'Not 24/7']
+let broken = 0
-
-const stats = { broken: 0 }
 
 async function main() {
-  console.info(`\nStarting...`)
+  log.start()
-  console.time('Done in')
-  if (config.debug) {
-    console.info(chalk.yellow(`INFO: Debug mode enabled\n`))
-  }
-  const playlists = parseIndex()
 
-  for (const playlist of playlists) {
+  log.print(`Parsing 'index.m3u'...`)
-    await loadPlaylist(playlist.url).then(checkStatus).then(savePlaylist).then(done)
-  }
-
-  finish()
-}
-
-function parseIndex() {
-  console.info(`Parsing 'index.m3u'...`)
   let playlists = parser.parseIndex()
   playlists = utils.filterPlaylists(playlists, config.country, config.exclude)
-  console.info(`Found ${playlists.length} playlist(s)\n`)
+  for (const playlist of playlists) {
+    await parser
+      .parsePlaylist(playlist.url)
+      .then(checkStatus)
+      .then(p => p.save())
+  }
 
-  return playlists
+  log.finish()
-}
-
-async function loadPlaylist(url) {
-  console.info(`Processing '${url}'...`)
-  return parser.parsePlaylist(url)
 }
 
 async function checkStatus(playlist) {
-  let bar
+  let bar = new ProgressBar(`Checking '${playlist.url}': [:bar] :current/:total (:percent) `, {
-  if (!config.debug) {
+    total: playlist.channels.length
-    bar = new ProgressBar('  Testing: [:bar] :current/:total (:percent) ', {
+  })
-      total: playlist.channels.length
+  const channels = []
-    })
-  }
-  const results = []
   const total = playlist.channels.length
   for (const [index, channel] of playlist.channels.entries()) {
     const current = index + 1
     const counter = chalk.gray(`[${current}/${total}]`)
-    if (bar) bar.tick()
+    bar.tick()
     if (
       (channel.status && ignore.map(i => i.toLowerCase()).includes(channel.status.toLowerCase())) ||
       (!channel.url.startsWith('http://') && !channel.url.startsWith('https://'))
     ) {
-      results.push(channel)
+      channels.push(channel)
-      if (config.debug) {
-        console.info(`  ${counter} ${chalk.green('online')} ${chalk.white(channel.url)}`)
-      }
     } else {
       const CancelToken = axios.CancelToken
       const source = CancelToken.source()
@@ -94,55 +73,27 @@ async function checkStatus(playlist) {
         .get(channel.url, { cancelToken: source.token })
         .then(() => {
           clearTimeout(timeout)
-          results.push(channel)
+          channels.push(channel)
-          if (config.debug) {
-            console.info(`  ${counter} ${chalk.green('online')} ${chalk.white(channel.url)}`)
-          }
         })
         .then(utils.sleep(config.delay))
         .catch(err => {
           clearTimeout(timeout)
           if (err.response && offlineStatusCodes.includes(err.response.status)) {
-            if (config.debug) {
+            broken++
-              console.info(`  ${counter} ${chalk.red('offline')} ${chalk.white(channel.url)}`)
-            }
-            stats.broken++
           } else {
-            results.push(channel)
+            channels.push(channel)
-            if (config.debug) {
-              console.info(`  ${counter} ${chalk.green('online')} ${chalk.white(channel.url)}`)
-            }
           }
         })
     }
   }
 
-  playlist.channels = results
+  if (playlist.channels.length !== channels.length) {
+    log.print(`File '${playlist.url}' has been updated\n`)
+    playlist.channels = channels
+    playlist.updated = true
+  }
 
   return playlist
 }
 
-async function savePlaylist(playlist) {
-  const original = utils.readFile(playlist.url)
-  const output = playlist.toString({ raw: true })
-
-  if (original === output) {
-    console.info(`No changes have been made.`)
-    return false
-  } else {
-    utils.createFile(playlist.url, output)
-    console.info(`Playlist has been updated. Removed ${stats.broken} links.`)
-  }
-
-  return true
-}
-
-async function done() {
-  console.info(` `)
-}
-
-function finish() {
-  console.timeEnd('Done in')
-}
-
 main()

scripts/detect-resolution.js

@@ -0,0 +1,110 @@
+const { program } = require('commander')
+const ProgressBar = require('progress')
+const axios = require('axios')
+const https = require('https')
+const parser = require('./helpers/parser')
+const utils = require('./helpers/utils')
+const log = require('./helpers/log')
+
+program
+  .usage('[OPTIONS]...')
+  .option('-c, --country <country>', 'Comma-separated list of country codes', '')
+  .option('-e, --exclude <exclude>', 'Comma-separated list of country codes to be excluded', '')
+  .option('--delay <delay>', 'Delay between parser requests', 1000)
+  .option('--timeout <timeout>', 'Set timeout for each request', 5000)
+  .parse(process.argv)
+
+const config = program.opts()
+const instance = axios.create({
+  timeout: config.timeout,
+  maxContentLength: 200000,
+  httpsAgent: new https.Agent({
+    rejectUnauthorized: false
+  })
+})
+
+async function main() {
+  log.start()
+
+  log.print(`Parsing 'index.m3u'...\n`)
+  let playlists = parser.parseIndex()
+  playlists = utils
+    .filterPlaylists(playlists, config.country, config.exclude)
+    .filter(i => i.url !== 'channels/unsorted.m3u')
+
+  for (const playlist of playlists) {
+    await parser
+      .parsePlaylist(playlist.url)
+      .then(detectResolution)
+      .then(p => p.save())
+  }
+
+  log.finish()
+}
+
+async function detectResolution(playlist) {
+  const channels = []
+  const bar = new ProgressBar(`Processing '${playlist.url}': [:bar] :current/:total (:percent) `, {
+    total: playlist.channels.length
+  })
+  let updated = false
+  for (const channel of playlist.channels) {
+    bar.tick()
+    if (!channel.resolution.height) {
+      const CancelToken = axios.CancelToken
+      const source = CancelToken.source()
+      const timeout = setTimeout(() => {
+        source.cancel()
+      }, config.timeout)
+
+      const response = await instance
+        .get(channel.url, { cancelToken: source.token })
+        .then(res => {
+          clearTimeout(timeout)
+
+          return res
+        })
+        .then(utils.sleep(config.delay))
+        .catch(err => {
+          clearTimeout(timeout)
+        })
+
+      if (response && response.status === 200) {
+        if (/^#EXTM3U/.test(response.data)) {
+          const resolution = parseResolution(response.data)
+          if (resolution) {
+            channel.resolution = resolution
+            updated = true
+          }
+        }
+      }
+    }
+
+    channels.push(channel)
+  }
+
+  if (updated) {
+    log.print(`File '${playlist.url}' has been updated\n`)
+    playlist.channels = channels
+    playlist.updated = true
+  }
+
+  return playlist
+}
+
+function parseResolution(string) {
+  const regex = /RESOLUTION=(\d+)x(\d+)/gm
+  const match = string.matchAll(regex)
+  const arr = Array.from(match).map(m => ({
+    width: parseInt(m[1]),
+    height: parseInt(m[2])
+  }))
+
+  return arr.length
+    ? arr.reduce(function (prev, current) {
+        return prev.height > current.height ? prev : current
+      })
+    : undefined
+}
+
+main()

scripts/filter.js

@@ -1,67 +1,43 @@
-const parser = require('./parser')
+const blacklist = require('./helpers/blacklist.json')
-const utils = require('./utils')
+const parser = require('./helpers/parser')
-const blacklist = require('./blacklist.json')
+const log = require('./helpers/log')
 
 async function main() {
-  const playlists = parseIndex()
+  log.start()
+
+  log.print(`Parsing 'index.m3u'...`)
+  const playlists = parser.parseIndex()
   for (const playlist of playlists) {
-    await loadPlaylist(playlist.url).then(removeBlacklisted).then(savePlaylist).then(done)
+    log.print(`\nProcessing '${playlist.url}'...`)
+    await parser
+      .parsePlaylist(playlist.url)
+      .then(removeBlacklisted)
+      .then(p => p.save())
   }
 
-  finish()
+  log.print('\n')
-}
+  log.finish()
-
-function parseIndex() {
-  console.info(`Parsing 'index.m3u'...`)
-  let playlists = parser.parseIndex()
-  console.info(`Found ${playlists.length} playlist(s)\n`)
-
-  return playlists
-}
-
-async function loadPlaylist(url) {
-  console.info(`Processing '${url}'...`)
-  return parser.parsePlaylist(url)
 }
 
 async function removeBlacklisted(playlist) {
-  console.info(`  Looking for blacklisted channels...`)
+  const channels = playlist.channels.filter(channel => {
-  playlist.channels = playlist.channels.filter(channel => {
+    return !blacklist.find(item => {
-    return !blacklist.find(i => {
+      const hasSameName =
-      const channelName = channel.name.toLowerCase()
+        item.name.toLowerCase() === channel.name.toLowerCase() ||
-      return (
+        item.aliases.map(alias => alias.toLowerCase()).includes(channel.name.toLowerCase())
-        (i.name.toLowerCase() === channelName ||
+      const fromSameCountry = channel.countries.find(c => c.code === item.country)
-          i.aliases.map(i => i.toLowerCase()).includes(channelName)) &&
+
-        i.country === channel.filename
+      return hasSameName && fromSameCountry
-      )
     })
   })
 
+  if (playlist.channels.length !== channels.length) {
+    log.print(`updated`)
+    playlist.channels = channels
+    playlist.updated = true
+  }
+
   return playlist
 }
 
-async function savePlaylist(playlist) {
-  console.info(`  Saving playlist...`)
-  const original = utils.readFile(playlist.url)
-  const output = playlist.toString({ raw: true })
-
-  if (original === output) {
-    console.info(`No changes have been made.`)
-    return false
-  } else {
-    utils.createFile(playlist.url, output)
-    console.info(`Playlist has been updated.`)
-  }
-
-  return true
-}
-
-async function done() {
-  console.info(` `)
-}
-
-function finish() {
-  console.info('Done.')
-}
-
 main()

scripts/format.js

@@ -1,152 +1,55 @@
-const { program } = require('commander')
+const parser = require('./helpers/parser')
-const parser = require('./parser')
+const utils = require('./helpers/utils')
-const utils = require('./utils')
+const file = require('./helpers/file')
-const axios = require('axios')
+const log = require('./helpers/log')
-const ProgressBar = require('progress')
-const https = require('https')
-
-program
-  .usage('[OPTIONS]...')
-  .option('-d, --debug', 'Debug mode')
-  .option('-r --resolution', 'Parse stream resolution')
-  .option('-c, --country <country>', 'Comma-separated list of country codes', '')
-  .option('-e, --exclude <exclude>', 'Comma-separated list of country codes to be excluded', '')
-  .option('--delay <delay>', 'Delay between parser requests', 1000)
-  .option('--timeout <timeout>', 'Set timeout for each request', 5000)
-  .parse(process.argv)
-
-const config = program.opts()
-
-const instance = axios.create({
-  timeout: config.timeout,
-  maxContentLength: 200000,
-  httpsAgent: new https.Agent({
-    rejectUnauthorized: false
-  })
-})
 
 async function main() {
-  console.info('Starting...')
+  log.start()
-  console.time('Done in')
-
-  const playlists = parseIndex()
 
+  log.print(`Parsing 'index.m3u'...`)
+  let playlists = parser.parseIndex().filter(i => i.url !== 'channels/unsorted.m3u')
   for (const playlist of playlists) {
-    await loadPlaylist(playlist.url)
+    log.print(`\nProcessing '${playlist.url}'...`)
-      .then(sortChannels)
+    await parser
-      .then(detectResolution)
+      .parsePlaylist(playlist.url)
-      .then(savePlaylist)
+      .then(formatPlaylist)
-      .then(done)
+      .then(playlist => {
-  }
+        if (file.read(playlist.url) !== playlist.toString()) {
-
+          log.print('updated')
-  finish()
+          playlist.updated = true
-}
-
-function parseIndex() {
-  console.info(`\nParsing 'index.m3u'...`)
-  let playlists = parser.parseIndex()
-  playlists = utils
-    .filterPlaylists(playlists, config.country, config.exclude)
-    .filter(i => i.url !== 'channels/unsorted.m3u')
-  console.info(`Found ${playlists.length} playlist(s)\n`)
-
-  return playlists
-}
-
-async function loadPlaylist(url) {
-  console.info(`Processing '${url}'...`)
-  return parser.parsePlaylist(url)
-}
-
-async function sortChannels(playlist) {
-  console.info(`  Sorting channels...`)
-  playlist.channels = utils.sortBy(playlist.channels, ['name', 'url'])
-
-  return playlist
-}
-
-async function detectResolution(playlist) {
-  if (!config.resolution) return playlist
-  console.log('  Detecting resolution...')
-  const bar = new ProgressBar('    Progress: [:bar] :current/:total (:percent) ', {
-    total: playlist.channels.length
-  })
-  const results = []
-  for (const channel of playlist.channels) {
-    bar.tick()
-    if (!channel.resolution.height) {
-      const CancelToken = axios.CancelToken
-      const source = CancelToken.source()
-      const timeout = setTimeout(() => {
-        source.cancel()
-      }, config.timeout)
-
-      const response = await instance
-        .get(channel.url, { cancelToken: source.token })
-        .then(res => {
-          clearTimeout(timeout)
-
-          return res
-        })
-        .then(utils.sleep(config.delay))
-        .catch(err => {
-          clearTimeout(timeout)
-        })
-
-      if (response && response.status === 200) {
-        if (/^#EXTM3U/.test(response.data)) {
-          const resolution = parseResolution(response.data)
-          if (resolution) {
-            channel.resolution = resolution
-          }
         }
-      }
-    }
 
-    results.push(channel)
+        playlist.save()
+      })
   }
 
-  playlist.channels = results
+  log.print('\n')
+  log.finish()
+}
+
+async function formatPlaylist(playlist) {
+  for (const channel of playlist.channels) {
+    const code = file.getBasename(playlist.url)
+    // add missing tvg-name
+    if (!channel.tvg.name && code !== 'unsorted' && channel.name) {
+      channel.tvg.name = channel.name.replace(/\"/gi, '')
+    }
+    // add missing tvg-id
+    if (!channel.tvg.id && code !== 'unsorted' && channel.tvg.name) {
+      const id = utils.name2id(channel.tvg.name)
+      channel.tvg.id = id ? `${id}.${code}` : ''
+    }
+    // add missing country
+    if (!channel.countries.length) {
+      const name = utils.code2name(code)
+      channel.countries = name ? [{ code, name }] : []
+      channel.tvg.country = channel.countries.map(c => c.code.toUpperCase()).join(';')
+    }
+    // update group-title
+    channel.group.title = channel.category
+  }
 
   return playlist
 }
 
-function parseResolution(string) {
-  const regex = /RESOLUTION=(\d+)x(\d+)/gm
-  const match = string.matchAll(regex)
-  const arr = Array.from(match).map(m => ({
-    width: parseInt(m[1]),
-    height: parseInt(m[2])
-  }))
-
-  return arr.length
-    ? arr.reduce(function (prev, current) {
-        return prev.height > current.height ? prev : current
-      })
-    : undefined
-}
-
-async function savePlaylist(playlist) {
-  const original = utils.readFile(playlist.url)
-  const output = playlist.toString()
-
-  if (original === output) {
-    console.info(`No changes have been made.`)
-    return false
-  } else {
-    utils.createFile(playlist.url, output)
-    console.info(`Playlist has been updated.`)
-  }
-
-  return true
-}
-
-async function done() {
-  console.info(` `)
-}
-
-function finish() {
-  console.timeEnd('Done in')
-}
-
 main()

scripts/generate.js

@@ -1,11 +1,11 @@
-const db = require('./db')
+const file = require('./helpers/file')
-const utils = require('./utils')
+const log = require('./helpers/log')
+const db = require('./helpers/db')
 
 const ROOT_DIR = './.gh-pages'
 
-db.load()
+async function main() {
-
+  await loadDatabase()
-function main() {
   createRootDirectory()
   createNoJekyllFile()
   generateIndex()
@@ -16,51 +16,56 @@ function main() {
   generateCountries()
   generateLanguages()
   generateChannelsJson()
-  finish()
+  showResults()
+}
+
+async function loadDatabase() {
+  log.print('Loading database...\n')
+  await db.load()
 }
 
 function createRootDirectory() {
-  console.log('Creating .gh-pages folder...')
+  log.print('Creating .gh-pages folder...\n')
-  utils.createDir(ROOT_DIR)
+  file.createDir(ROOT_DIR)
 }
 
 function createNoJekyllFile() {
-  console.log('Creating .nojekyll...')
+  log.print('Creating .nojekyll...\n')
-  utils.createFile(`${ROOT_DIR}/.nojekyll`)
+  file.create(`${ROOT_DIR}/.nojekyll`)
 }
 
 function generateIndex() {
-  console.log('Generating index.m3u...')
+  log.print('Generating index.m3u...\n')
   const filename = `${ROOT_DIR}/index.m3u`
-  utils.createFile(filename, '#EXTM3U\n')
+  file.create(filename, '#EXTM3U\n')
 
   const nsfwFilename = `${ROOT_DIR}/index.nsfw.m3u`
-  utils.createFile(nsfwFilename, '#EXTM3U\n')
+  file.create(nsfwFilename, '#EXTM3U\n')
 
   const channels = db.channels.sortBy(['name', 'url']).removeDuplicates().get()
   for (const channel of channels) {
     if (!channel.isNSFW()) {
-      utils.appendToFile(filename, channel.toString())
+      file.append(filename, channel.toString())
     }
-    utils.appendToFile(nsfwFilename, channel.toString())
+    file.append(nsfwFilename, channel.toString())
   }
 }
 
 function generateCategoryIndex() {
-  console.log('Generating index.category.m3u...')
+  log.print('Generating index.category.m3u...\n')
   const filename = `${ROOT_DIR}/index.category.m3u`
-  utils.createFile(filename, '#EXTM3U\n')
+  file.create(filename, '#EXTM3U\n')
 
   const channels = db.channels.sortBy(['category', 'name', 'url']).removeDuplicates().get()
   for (const channel of channels) {
-    utils.appendToFile(filename, channel.toString())
+    file.append(filename, channel.toString())
   }
 }
 
 function generateCountryIndex() {
-  console.log('Generating index.country.m3u...')
+  log.print('Generating index.country.m3u...\n')
   const filename = `${ROOT_DIR}/index.country.m3u`
-  utils.createFile(filename, '#EXTM3U\n')
+  file.create(filename, '#EXTM3U\n')
 
   for (const country of [{ code: 'undefined' }, ...db.countries.sortBy(['name']).all()]) {
     const channels = db.channels
@@ -69,21 +74,21 @@ function generateCountryIndex() {
       .removeDuplicates()
       .get()
     for (const channel of channels) {
-      const category = channel.category
+      const groupTitle = channel.group.title
       const nsfw = channel.isNSFW()
-      channel.category = country.name || ''
+      channel.group.title = country.name || ''
       if (!nsfw) {
-        utils.appendToFile(filename, channel.toString())
+        file.append(filename, channel.toString())
       }
-      channel.category = category
+      channel.group.title = groupTitle
     }
   }
 }
 
 function generateLanguageIndex() {
-  console.log('Generating index.language.m3u...')
+  log.print('Generating index.language.m3u...\n')
   const filename = `${ROOT_DIR}/index.language.m3u`
-  utils.createFile(filename, '#EXTM3U\n')
+  file.create(filename, '#EXTM3U\n')
 
   for (const language of [{ code: 'undefined' }, ...db.languages.sortBy(['name']).all()]) {
     const channels = db.channels
@@ -92,25 +97,25 @@ function generateLanguageIndex() {
       .removeDuplicates()
       .get()
     for (const channel of channels) {
-      const category = channel.category
+      const groupTitle = channel.group.title
       const nsfw = channel.isNSFW()
-      channel.category = language.name || ''
+      channel.group.title = language.name || ''
       if (!nsfw) {
-        utils.appendToFile(filename, channel.toString())
+        file.append(filename, channel.toString())
       }
-      channel.category = category
+      channel.group.title = groupTitle
     }
   }
 }
 
 function generateCategories() {
-  console.log(`Generating /categories...`)
+  log.print(`Generating /categories...\n`)
   const outputDir = `${ROOT_DIR}/categories`
-  utils.createDir(outputDir)
+  file.createDir(outputDir)
 
   for (const category of [...db.categories.all(), { id: 'other' }]) {
     const filename = `${outputDir}/${category.id}.m3u`
-    utils.createFile(filename, '#EXTM3U\n')
+    file.create(filename, '#EXTM3U\n')
 
     const channels = db.channels
       .sortBy(['name', 'url'])
@@ -118,19 +123,19 @@ function generateCategories() {
       .removeDuplicates()
       .get()
     for (const channel of channels) {
-      utils.appendToFile(filename, channel.toString())
+      file.append(filename, channel.toString())
     }
   }
 }
 
 function generateCountries() {
-  console.log(`Generating /countries...`)
+  log.print(`Generating /countries...\n`)
   const outputDir = `${ROOT_DIR}/countries`
-  utils.createDir(outputDir)
+  file.createDir(outputDir)
 
   for (const country of [...db.countries.all(), { code: 'undefined' }]) {
     const filename = `${outputDir}/${country.code}.m3u`
-    utils.createFile(filename, '#EXTM3U\n')
+    file.create(filename, '#EXTM3U\n')
 
     const channels = db.channels
       .sortBy(['name', 'url'])
@@ -139,20 +144,20 @@ function generateCountries() {
       .get()
     for (const channel of channels) {
       if (!channel.isNSFW()) {
-        utils.appendToFile(filename, channel.toString())
+        file.append(filename, channel.toString())
       }
     }
   }
 }
 
 function generateLanguages() {
-  console.log(`Generating /languages...`)
+  log.print(`Generating /languages...\n`)
   const outputDir = `${ROOT_DIR}/languages`
-  utils.createDir(outputDir)
+  file.createDir(outputDir)
 
   for (const language of [...db.languages.all(), { code: 'undefined' }]) {
     const filename = `${outputDir}/${language.code}.m3u`
-    utils.createFile(filename, '#EXTM3U\n')
+    file.create(filename, '#EXTM3U\n')
 
     const channels = db.channels
       .sortBy(['name', 'url'])
@@ -161,25 +166,25 @@ function generateLanguages() {
       .get()
     for (const channel of channels) {
       if (!channel.isNSFW()) {
-        utils.appendToFile(filename, channel.toString())
+        file.append(filename, channel.toString())
       }
     }
   }
 }
 
 function generateChannelsJson() {
-  console.log('Generating channels.json...')
+  log.print('Generating channels.json...\n')
   const filename = `${ROOT_DIR}/channels.json`
   const channels = db.channels
     .sortBy(['name', 'url'])
     .get()
     .map(c => c.toObject())
-  utils.createFile(filename, JSON.stringify(channels))
+  file.create(filename, JSON.stringify(channels))
 }
 
-function finish() {
+function showResults() {
-  console.log(
+  log.print(
-    `\nTotal: ${db.channels.count()} channels, ${db.countries.count()} countries, ${db.languages.count()} languages, ${db.categories.count()} categories.`
+    `Total: ${db.channels.count()} channels, ${db.countries.count()} countries, ${db.languages.count()} languages, ${db.categories.count()} categories.\n`
   )
 }
 

scripts/helpers/Channel.js

@@ -0,0 +1,145 @@
+const categories = require('./categories')
+const utils = require('./utils')
+const file = require('./file')
+
+const sfwCategories = categories.filter(c => !c.nsfw).map(c => c.name)
+const nsfwCategories = categories.filter(c => c.nsfw).map(c => c.name)
+
+module.exports = class Channel {
+  constructor(data) {
+    this.raw = data.raw
+    this.tvg = data.tvg
+    this.http = data.http
+    this.url = data.url
+    this.logo = data.tvg.logo
+    this.group = data.group
+    this.name = this.parseName(data.name)
+    this.status = this.parseStatus(data.name)
+    this.resolution = this.parseResolution(data.name)
+    this.category = this.parseCategory(data.group.title)
+    this.countries = this.parseCountries(data.tvg.country)
+    this.languages = this.parseLanguages(data.tvg.language)
+  }
+
+  parseName(title) {
+    return title
+      .trim()
+      .split(' ')
+      .map(s => s.trim())
+      .filter(s => {
+        return !/\[|\]/i.test(s) && !/\((\d+)P\)/i.test(s)
+      })
+      .join(' ')
+  }
+
+  parseStatus(title) {
+    const match = title.match(/\[(.*)\]/i)
+    return match ? match[1] : null
+  }
+
+  parseResolution(title) {
+    const match = title.match(/\((\d+)P\)/i)
+    const height = match ? parseInt(match[1]) : null
+
+    return { width: null, height }
+  }
+
+  parseCategory(string) {
+    const category = categories.find(c => c.id === string.toLowerCase())
+    if (!category) return ''
+
+    return category.name
+  }
+
+  parseCountries(string) {
+    const list = string.split(';')
+    return list
+      .reduce((acc, curr) => {
+        const codes = utils.region2codes(curr)
+        if (codes.length) {
+          for (let code of codes) {
+            if (!acc.includes(code)) {
+              acc.push(code)
+            }
+          }
+        } else {
+          acc.push(curr)
+        }
+
+        return acc
+      }, [])
+      .map(code => {
+        const name = code ? utils.code2name(code) : null
+        if (!name) return null
+
+        return { code: code.toLowerCase(), name }
+      })
+      .filter(c => c)
+  }
+
+  parseLanguages(string) {
+    const list = string.split(';')
+    return list
+      .map(name => {
+        const code = name ? utils.language2code(name) : null
+        if (!code) return null
+
+        return { code, name }
+      })
+      .filter(l => l)
+  }
+
+  isSFW() {
+    return sfwCategories.includes(this.category)
+  }
+
+  isNSFW() {
+    return nsfwCategories.includes(this.category)
+  }
+
+  getInfo() {
+    let info = `-1 tvg-id="${this.tvg.id}" tvg-name="${this.tvg.name}" tvg-country="${this.tvg.country}" tvg-language="${this.tvg.language}" tvg-logo="${this.logo}"`
+
+    info += ` group-title="${this.group.title}",${this.name}`
+
+    if (this.resolution.height) {
+      info += ` (${this.resolution.height}p)`
+    }
+
+    if (this.status) {
+      info += ` [${this.status}]`
+    }
+
+    if (this.http['referrer']) {
+      info += `\n#EXTVLCOPT:http-referrer=${this.http['referrer']}`
+    }
+
+    if (this.http['user-agent']) {
+      info += `\n#EXTVLCOPT:http-user-agent=${this.http['user-agent']}`
+    }
+
+    return info
+  }
+
+  toString(raw = false) {
+    if (raw) return this.raw + '\n'
+
+    return '#EXTINF:' + this.getInfo() + '\n' + this.url + '\n'
+  }
+
+  toObject() {
+    return {
+      name: this.name,
+      logo: this.logo || null,
+      url: this.url,
+      category: this.category || null,
+      languages: this.languages,
+      countries: this.countries,
+      tvg: {
+        id: this.tvg.id || null,
+        name: this.tvg.name || null,
+        url: this.tvg.url || null
+      }
+    }
+  }
+}

scripts/helpers/Playlist.js

@@ -0,0 +1,37 @@
+const Channel = require('./Channel')
+const file = require('./file')
+
+module.exports = class Playlist {
+  constructor({ header, items, url, name, country }) {
+    this.url = url
+    this.name = name
+    this.country = country
+    this.header = header
+    this.channels = items.map(item => new Channel(item)).filter(channel => channel.url)
+    this.updated = false
+  }
+
+  toString(options = {}) {
+    const config = { raw: false, ...options }
+    let parts = ['#EXTM3U']
+    for (let key in this.header.attrs) {
+      let value = this.header.attrs[key]
+      if (value) {
+        parts.push(`${key}="${value}"`)
+      }
+    }
+
+    let output = `${parts.join(' ')}\n`
+    for (let channel of this.channels) {
+      output += channel.toString(config.raw)
+    }
+
+    return output
+  }
+
+  save() {
+    if (this.updated) {
+      file.create(this.url, this.toString())
+    }
+  }
+}

scripts/helpers/db.js

@@ -2,14 +2,12 @@ const categories = require('./categories')
 const parser = require('./parser')
 const utils = require('./utils')
 
-const sfwCategories = categories.filter(c => !c.nsfw).map(c => c.name)
-
 const db = {}
 
-db.load = function () {
+db.load = async function () {
   const items = parser.parseIndex()
   for (const item of items) {
-    const playlist = parser.parsePlaylist(item.url)
+    const playlist = await parser.parsePlaylist(item.url)
     db.playlists.add(playlist)
     for (const channel of playlist.channels) {
       db.channels.add(channel)
@@ -107,9 +105,6 @@ db.channels = {
   all() {
     return this.list
   },
-  sfw() {
-    return this.list.filter(i => sfwCategories.includes(i.category))
-  },
   forCountry(country) {
     this.filter = {
       field: 'countries',

scripts/helpers/file.js

@@ -0,0 +1,38 @@
+const markdownInclude = require('markdown-include')
+const path = require('path')
+const fs = require('fs')
+
+const rootPath = path.resolve(__dirname) + '/../../'
+const file = {}
+
+file.getBasename = function (filename) {
+  return path.basename(filename, path.extname(filename))
+}
+
+file.getFilename = function (filename) {
+  return path.parse(filename).name
+}
+
+file.createDir = function (dir) {
+  if (!fs.existsSync(dir)) {
+    fs.mkdirSync(dir)
+  }
+}
+
+file.read = function (filename) {
+  return fs.readFileSync(rootPath + filename, { encoding: 'utf8' })
+}
+
+file.append = function (filename, data) {
+  fs.appendFileSync(rootPath + filename, data)
+}
+
+file.create = function (filename, data = '') {
+  fs.writeFileSync(rootPath + filename, data)
+}
+
+file.compileMarkdown = function (filename) {
+  markdownInclude.compileFiles(rootPath + filename)
+}
+
+module.exports = file

scripts/helpers/log.js

@@ -0,0 +1,16 @@
+const log = {}
+
+log.print = function (string) {
+  process.stdout.write(string)
+}
+
+log.start = function () {
+  this.print('Starting...\n')
+  console.time('Done in')
+}
+
+log.finish = function () {
+  console.timeEnd('Done in')
+}
+
+module.exports = log

scripts/helpers/parser.js

@@ -0,0 +1,24 @@
+const playlistParser = require('iptv-playlist-parser')
+const Playlist = require('./Playlist')
+const utils = require('./utils')
+const file = require('./file')
+
+const parser = {}
+
+parser.parseIndex = function () {
+  const content = file.read('index.m3u')
+  const result = playlistParser.parse(content)
+
+  return result.items
+}
+
+parser.parsePlaylist = async function (url) {
+  const content = file.read(url)
+  const result = playlistParser.parse(content)
+  const name = file.getFilename(url)
+  const country = utils.code2name(name)
+
+  return new Playlist({ header: result.header, items: result.items, url, country, name })
+}
+
+module.exports = parser

scripts/helpers/utils.js

@@ -1,21 +1,15 @@
-const fs = require('fs')
-const path = require('path')
-const axios = require('axios')
-const zlib = require('zlib')
-const urlParser = require('url')
 const escapeStringRegexp = require('escape-string-regexp')
-const markdownInclude = require('markdown-include')
-const iso6393 = require('@freearhey/iso-639-3')
 const transliteration = require('transliteration')
-const regions = require('./regions')
+const iso6393 = require('@freearhey/iso-639-3')
 const categories = require('./categories')
+const regions = require('./regions')
+
+const utils = {}
 const intlDisplayNames = new Intl.DisplayNames(['en'], {
   style: 'narrow',
   type: 'region'
 })
 
-const utils = {}
-
 utils.name2id = function (name) {
   return transliteration
     .transliterate(name)
@@ -66,36 +60,29 @@ utils.sortBy = function (arr, fields) {
     for (let field of fields) {
       let propA = a[field] ? a[field].toLowerCase() : ''
       let propB = b[field] ? b[field].toLowerCase() : ''
-
+      if (propA === 'undefined') return 1
-      if (propA === 'undefined') {
+      if (propB === 'undefined') return -1
-        return 1
+      if (propA === 'other') return 1
-      }
+      if (propB === 'other') return -1
-
+      if (propA < propB) return -1
-      if (propB === 'undefined') {
+      if (propA > propB) return 1
-        return -1
-      }
-
-      if (propA === 'other') {
-        return 1
-      }
-
-      if (propB === 'other') {
-        return -1
-      }
-
-      if (propA < propB) {
-        return -1
-      }
-      if (propA > propB) {
-        return 1
-      }
     }
     return 0
   })
 }
 
-utils.getBasename = function (filename) {
+utils.escapeStringRegexp = function (scring) {
-  return path.basename(filename, path.extname(filename))
+  return escapeStringRegexp(string)
+}
+
+utils.sleep = function (ms) {
+  return function (x) {
+    return new Promise(resolve => setTimeout(() => resolve(x), ms))
+  }
+}
+
+utils.removeProtocol = function (string) {
+  return string.replace(/(^\w+:|^)\/\//, '')
 }
 
 utils.filterPlaylists = function (arr, include = '', exclude = '') {
@@ -114,75 +101,4 @@ utils.filterPlaylists = function (arr, include = '', exclude = '') {
   return arr
 }
 
-utils.generateTable = function (data, options) {
-  let output = '<table>\n'
-
-  output += '\t<thead>\n\t\t<tr>'
-  for (let column of options.columns) {
-    output += `<th align="${column.align}">${column.name}</th>`
-  }
-  output += '</tr>\n\t</thead>\n'
-
-  output += '\t<tbody>\n'
-  for (let item of data) {
-    output += '\t\t<tr>'
-    let i = 0
-    for (let prop in item) {
-      const column = options.columns[i]
-      let nowrap = column.nowrap
-      let align = column.align
-      output += `<td align="${align}"${nowrap ? ' nowrap' : ''}>${item[prop]}</td>`
-      i++
-    }
-    output += '</tr>\n'
-  }
-  output += '\t</tbody>\n'
-
-  output += '</table>'
-
-  return output
-}
-
-utils.createDir = function (dir) {
-  if (!fs.existsSync(dir)) {
-    fs.mkdirSync(dir)
-  }
-}
-
-utils.readFile = function (filename) {
-  return fs.readFileSync(path.resolve(__dirname) + `/../${filename}`, { encoding: 'utf8' })
-}
-
-utils.appendToFile = function (filename, data) {
-  fs.appendFileSync(path.resolve(__dirname) + '/../' + filename, data)
-}
-
-utils.compileMarkdown = function (filepath) {
-  return markdownInclude.compileFiles(path.resolve(__dirname, filepath))
-}
-
-utils.escapeStringRegexp = function (scring) {
-  return escapeStringRegexp(string)
-}
-
-utils.createFile = function (filename, data = '') {
-  fs.writeFileSync(path.resolve(__dirname) + '/../' + filename, data)
-}
-
-utils.writeToLog = function (country, msg, url) {
-  var now = new Date()
-  var line = `${country}: ${msg} '${url}'`
-  this.appendToFile('error.log', now.toISOString() + ' ' + line + '\n')
-}
-
-utils.sleep = function (ms) {
-  return function (x) {
-    return new Promise(resolve => setTimeout(() => resolve(x), ms))
-  }
-}
-
-utils.removeProtocol = function (string) {
-  return string.replace(/(^\w+:|^)\/\//, '')
-}
-
 module.exports = utils

scripts/parser.js

@@ -1,244 +0,0 @@
-const playlistParser = require('iptv-playlist-parser')
-const utils = require('./utils')
-const categories = require('./categories')
-const path = require('path')
-
-const sfwCategories = categories.filter(c => !c.nsfw).map(c => c.name)
-const nsfwCategories = categories.filter(c => c.nsfw).map(c => c.name)
-
-const parser = {}
-
-parser.parseIndex = function () {
-  const content = utils.readFile('index.m3u')
-  const result = playlistParser.parse(content)
-
-  return result.items
-}
-
-parser.parsePlaylist = function (filename) {
-  const content = utils.readFile(filename)
-  const result = playlistParser.parse(content)
-  const name = path.parse(filename).name
-  const country = utils.code2name(name)
-
-  return new Playlist({ header: result.header, items: result.items, url: filename, country, name })
-}
-
-class Playlist {
-  constructor({ header, items, url, name, country }) {
-    this.url = url
-    this.name = name
-    this.country = country
-    this.header = header
-    this.channels = items
-      .map(item => new Channel({ data: item, header, sourceUrl: url }))
-      .filter(channel => channel.url)
-  }
-
-  toString(options = {}) {
-    const config = { raw: false, ...options }
-    let parts = ['#EXTM3U']
-    for (let key in this.header.attrs) {
-      let value = this.header.attrs[key]
-      if (value) {
-        parts.push(`${key}="${value}"`)
-      }
-    }
-
-    let output = `${parts.join(' ')}\n`
-    for (let channel of this.channels) {
-      output += channel.toString(config.raw)
-    }
-
-    return output
-  }
-}
-
-class Channel {
-  constructor({ data, header, sourceUrl }) {
-    this.parseData(data)
-
-    this.filename = utils.getBasename(sourceUrl)
-    if (!this.countries.length) {
-      const countryName = utils.code2name(this.filename)
-      this.countries = countryName ? [{ code: this.filename, name: countryName }] : []
-      this.tvg.country = this.countries.map(c => c.code.toUpperCase()).join(';')
-    }
-  }
-
-  parseData(data) {
-    const title = this.parseTitle(data.name)
-
-    this.tvg = data.tvg
-    this.http = data.http
-    this.url = data.url
-    this.logo = data.tvg.logo
-    this.name = title.channelName
-    this.status = title.streamStatus
-    this.resolution = title.streamResolution
-    this.countries = this.parseCountries(data.tvg.country)
-    this.languages = this.parseLanguages(data.tvg.language)
-    this.category = this.parseCategory(data.group.title)
-    this.raw = data.raw
-  }
-
-  parseCountries(string) {
-    let arr = string
-      .split(';')
-      .reduce((acc, curr) => {
-        const codes = utils.region2codes(curr)
-        if (codes.length) {
-          for (let code of codes) {
-            if (!acc.includes(code)) {
-              acc.push(code)
-            }
-          }
-        } else {
-          acc.push(curr)
-        }
-
-        return acc
-      }, [])
-      .filter(code => code && utils.code2name(code))
-
-    return arr.map(code => {
-      return { code: code.toLowerCase(), name: utils.code2name(code) }
-    })
-  }
-
-  parseLanguages(string) {
-    return string
-      .split(';')
-      .map(name => {
-        const code = name ? utils.language2code(name) : null
-        if (!code) return null
-
-        return {
-          code,
-          name
-        }
-      })
-      .filter(l => l)
-  }
-
-  parseCategory(string) {
-    const category = categories.find(c => c.id === string.toLowerCase())
-
-    return category ? category.name : ''
-  }
-
-  parseTitle(title) {
-    const channelName = title
-      .trim()
-      .split(' ')
-      .map(s => s.trim())
-      .filter(s => {
-        return !/\[|\]/i.test(s) && !/\((\d+)P\)/i.test(s)
-      })
-      .join(' ')
-
-    const streamStatusMatch = title.match(/\[(.*)\]/i)
-    const streamStatus = streamStatusMatch ? streamStatusMatch[1] : null
-
-    const streamResolutionMatch = title.match(/\((\d+)P\)/i)
-    const streamResolutionHeight = streamResolutionMatch ? parseInt(streamResolutionMatch[1]) : null
-    const streamResolution = { width: null, height: streamResolutionHeight }
-
-    return { channelName, streamStatus, streamResolution }
-  }
-
-  get tvgCountry() {
-    return this.tvg.country
-      .split(';')
-      .map(code => utils.code2name(code))
-      .join(';')
-  }
-
-  get tvgLanguage() {
-    return this.tvg.language
-  }
-
-  get tvgUrl() {
-    return this.tvg.id && this.tvg.url ? this.tvg.url : ''
-  }
-
-  get tvgId() {
-    if (this.tvg.id) {
-      return this.tvg.id
-    } else if (this.filename !== 'unsorted') {
-      const id = utils.name2id(this.tvgName)
-
-      return id ? `${id}.${this.filename}` : ''
-    }
-
-    return ''
-  }
-
-  get tvgName() {
-    if (this.tvg.name) {
-      return this.tvg.name
-    } else if (this.filename !== 'unsorted') {
-      return this.name.replace(/\"/gi, '')
-    }
-
-    return ''
-  }
-
-  getInfo() {
-    this.tvg.country = this.tvg.country.toUpperCase()
-
-    let info = `-1 tvg-id="${this.tvgId}" tvg-name="${this.tvgName}" tvg-country="${this.tvg.country}" tvg-language="${this.tvg.language}" tvg-logo="${this.logo}"`
-
-    info += ` group-title="${this.category}",${this.name}`
-
-    if (this.resolution.height) {
-      info += ` (${this.resolution.height}p)`
-    }
-
-    if (this.status) {
-      info += ` [${this.status}]`
-    }
-
-    if (this.http['referrer']) {
-      info += `\n#EXTVLCOPT:http-referrer=${this.http['referrer']}`
-    }
-
-    if (this.http['user-agent']) {
-      info += `\n#EXTVLCOPT:http-user-agent=${this.http['user-agent']}`
-    }
-
-    return info
-  }
-
-  toString(raw = false) {
-    if (raw) return this.raw + '\n'
-
-    return '#EXTINF:' + this.getInfo() + '\n' + this.url + '\n'
-  }
-
-  toObject() {
-    return {
-      name: this.name,
-      logo: this.logo || null,
-      url: this.url,
-      category: this.category || null,
-      languages: this.languages,
-      countries: this.countries,
-      tvg: {
-        id: this.tvgId || null,
-        name: this.tvgName || null,
-        url: this.tvgUrl || null
-      }
-    }
-  }
-
-  isSFW() {
-    return sfwCategories.includes(this.category)
-  }
-
-  isNSFW() {
-    return nsfwCategories.includes(this.category)
-  }
-}
-
-module.exports = parser

scripts/remove-duplicates.js

@@ -1,52 +1,42 @@
-const parser = require('./parser')
+const parser = require('./helpers/parser')
-const utils = require('./utils')
+const utils = require('./helpers/utils')
+const log = require('./helpers/log')
 
 let globalBuffer = []
 
 async function main() {
-  const playlists = parseIndex()
+  log.start()
 
+  log.print(`Parsing 'index.m3u'...`)
+  const playlists = parser.parseIndex().filter(i => i.url !== 'channels/unsorted.m3u')
   for (const playlist of playlists) {
-    await loadPlaylist(playlist.url)
+    log.print(`\nProcessing '${playlist.url}'...`)
+    await parser
+      .parsePlaylist(playlist.url)
       .then(addToBuffer)
       .then(removeDuplicates)
-      .then(savePlaylist)
+      .then(p => p.save())
-      .then(done)
   }
 
   if (playlists.length) {
-    await loadPlaylist('channels/unsorted.m3u')
+    log.print(`\nProcessing 'channels/unsorted.m3u'...`)
+    await parser
+      .parsePlaylist('channels/unsorted.m3u')
       .then(removeUnsortedDuplicates)
-      .then(savePlaylist)
+      .then(p => p.save())
-      .then(done)
   }
 
-  finish()
+  log.print('\n')
-}
+  log.finish()
-
-function parseIndex() {
-  console.info(`Parsing 'index.m3u'...`)
-  let playlists = parser.parseIndex()
-  playlists = playlists.filter(i => i.url !== 'channels/unsorted.m3u')
-  console.info(`Found ${playlists.length} playlist(s)\n`)
-
-  return playlists
-}
-
-async function loadPlaylist(url) {
-  console.info(`Processing '${url}'...`)
-  return parser.parsePlaylist(url)
 }
 
 async function addToBuffer(playlist) {
-  if (playlist.url === 'channels/unsorted.m3u') return playlist
   globalBuffer = globalBuffer.concat(playlist.channels)
 
   return playlist
 }
 
 async function removeDuplicates(playlist) {
-  console.info(`  Looking for duplicates...`)
   let buffer = {}
   const channels = playlist.channels.filter(i => {
     const url = utils.removeProtocol(i.url)
@@ -58,13 +48,16 @@ async function removeDuplicates(playlist) {
     return result
   })
 
-  playlist.channels = channels
+  if (playlist.channels.length !== channels.length) {
+    log.print('updated')
+    playlist.channels = channels
+    playlist.updated = true
+  }
 
   return playlist
 }
 
 async function removeUnsortedDuplicates(playlist) {
-  console.info(`  Looking for duplicates...`)
   // locally
   let buffer = {}
   let channels = playlist.channels.filter(i => {
@@ -74,37 +67,18 @@ async function removeUnsortedDuplicates(playlist) {
 
     return result
   })
+
   // globally
   const urls = globalBuffer.map(i => utils.removeProtocol(i.url))
   channels = channels.filter(i => !urls.includes(utils.removeProtocol(i.url)))
-  if (channels.length === playlist.channels.length) return playlist
 
-  playlist.channels = channels
+  if (channels.length !== playlist.channels.length) {
+    log.print('updated')
+    playlist.channels = channels
+    playlist.updated = true
+  }
 
   return playlist
 }
 
-async function savePlaylist(playlist) {
-  const original = utils.readFile(playlist.url)
-  const output = playlist.toString({ raw: true })
-
-  if (original === output) {
-    console.info(`No changes have been made.`)
-    return false
-  } else {
-    utils.createFile(playlist.url, output)
-    console.info(`Playlist has been updated.`)
-  }
-
-  return true
-}
-
-async function done() {
-  console.info(` `)
-}
-
-function finish() {
-  console.info('Done.')
-}
-
 main()

scripts/sort.js

@@ -0,0 +1,35 @@
+const parser = require('./helpers/parser')
+const utils = require('./helpers/utils')
+const log = require('./helpers/log')
+
+async function main() {
+  log.start()
+
+  log.print(`Parsing 'index.m3u'...`)
+  let playlists = parser.parseIndex().filter(i => i.url !== 'channels/unsorted.m3u')
+  for (const playlist of playlists) {
+    log.print(`\nProcessing '${playlist.url}'...`)
+    await parser
+      .parsePlaylist(playlist.url)
+      .then(sortChannels)
+      .then(p => p.save())
+  }
+
+  log.print('\n')
+  log.finish()
+}
+
+async function sortChannels(playlist) {
+  const channels = [...playlist.channels]
+  utils.sortBy(channels, ['name', 'url'])
+
+  if (JSON.stringify(channels) !== JSON.stringify(playlist.channels)) {
+    log.print('updated')
+    playlist.channels = channels
+    playlist.updated = true
+  }
+
+  return playlist
+}
+
+main()

scripts/update-readme.js

@@ -1,20 +1,25 @@
-const utils = require('./utils')
+const utils = require('./helpers/utils')
-const db = require('./db')
+const file = require('./helpers/file')
-const parser = require('./parser')
+const log = require('./helpers/log')
+const db = require('./helpers/db')
 
-db.load()
+async function main() {
-
+  log.start()
-function main() {
+  await loadDatabase()
-  start()
   generateCategoriesTable()
   generateCountriesTable()
   generateLanguagesTable()
   generateReadme()
-  finish()
+  log.finish()
+}
+
+async function loadDatabase() {
+  log.print('Loading database...\n')
+  await db.load()
 }
 
 function generateCategoriesTable() {
-  console.log(`Generating categories table...`)
+  log.print('Generating categories table...\n')
 
   const categories = []
   for (const category of [...db.categories.all(), { name: 'Other', id: 'other' }]) {
@@ -25,7 +30,7 @@ function generateCategoriesTable() {
     })
   }
 
-  const table = utils.generateTable(categories, {
+  const table = generateTable(categories, {
     columns: [
       { name: 'Category', align: 'left' },
       { name: 'Channels', align: 'right' },
@@ -33,11 +38,11 @@ function generateCategoriesTable() {
     ]
   })
 
-  utils.createFile('./.readme/_categories.md', table)
+  file.create('./.readme/_categories.md', table)
 }
 
 function generateCountriesTable() {
-  console.log(`Generating countries table...`)
+  log.print('Generating countries table...\n')
 
   const countries = []
   for (const country of [
@@ -53,7 +58,7 @@ function generateCountriesTable() {
     })
   }
 
-  const table = utils.generateTable(countries, {
+  const table = generateTable(countries, {
     columns: [
       { name: 'Country', align: 'left' },
       { name: 'Channels', align: 'right' },
@@ -61,11 +66,11 @@ function generateCountriesTable() {
     ]
   })
 
-  utils.createFile('./.readme/_countries.md', table)
+  file.create('./.readme/_countries.md', table)
 }
 
 function generateLanguagesTable() {
-  console.log(`Generating languages table...`)
+  log.print('Generating languages table...\n')
   const languages = []
 
   for (const language of [
@@ -79,7 +84,7 @@ function generateLanguagesTable() {
     })
   }
 
-  const table = utils.generateTable(languages, {
+  const table = generateTable(languages, {
     columns: [
       { name: 'Language', align: 'left' },
       { name: 'Channels', align: 'right' },
@@ -87,20 +92,41 @@ function generateLanguagesTable() {
     ]
   })
 
-  utils.createFile('./.readme/_languages.md', table)
+  file.create('./.readme/_languages.md', table)
+}
+
+function generateTable(data, options) {
+  let output = '<table>\n'
+
+  output += '\t<thead>\n\t\t<tr>'
+  for (let column of options.columns) {
+    output += `<th align="${column.align}">${column.name}</th>`
+  }
+  output += '</tr>\n\t</thead>\n'
+
+  output += '\t<tbody>\n'
+  for (let item of data) {
+    output += '\t\t<tr>'
+    let i = 0
+    for (let prop in item) {
+      const column = options.columns[i]
+      let nowrap = column.nowrap
+      let align = column.align
+      output += `<td align="${align}"${nowrap ? ' nowrap' : ''}>${item[prop]}</td>`
+      i++
+    }
+    output += '</tr>\n'
+  }
+  output += '\t</tbody>\n'
+
+  output += '</table>'
+
+  return output
 }
 
 function generateReadme() {
-  console.log(`Generating README.md...`)
+  log.print('Generating README.md...\n')
-  utils.compileMarkdown('../.readme/config.json')
+  file.compileMarkdown('.readme/config.json')
-}
-
-function start() {
-  console.log(`Starting...`)
-}
-
-function finish() {
-  console.log(`Done.`)
 }
 
 main()