Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8,097 changes: 4,039 additions & 4,058 deletions package-lock.json

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion package.json
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@
"@freearhey/core": "^0.14.3",
"@freearhey/storage-js": "^0.1.0",
"@inquirer/prompts": "^7.8.0",
"@iptv-org/sdk": "^1.4.0",
"@iptv-org/sdk": "^1.5.0",
"@octokit/core": "^7.0.3",
"@octokit/plugin-paginate-graphql": "^6.0.0",
"@octokit/plugin-paginate-rest": "^13.1.1",
Expand Down
7 changes: 6 additions & 1 deletion scripts/api.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ const data = {
feedsKeyByStreamId: new Dictionary<sdk.Models.Feed>(),
feedsGroupedByChannel: new Dictionary<sdk.Models.Feed[]>(),
blocklistRecordsGroupedByChannel: new Dictionary<sdk.Models.BlocklistRecord[]>(),
guidesGroupedByStreamId: new Dictionary<sdk.Models.Guide[]>(),
categories: new Collection<sdk.Models.Category>(),
countries: new Collection<sdk.Models.Country>(),
subdivisions: new Collection<sdk.Models.Subdivision>(),
Expand All @@ -37,7 +38,8 @@ async function loadData() {
subdivisions,
cities,
regions,
blocklist
blocklist,
guides
} = dataManager.getProcessedData()

const searchableData = channels.map((channel: sdk.Models.Channel) => channel.getSearchable())
Expand All @@ -57,6 +59,9 @@ async function loadData() {
data.blocklistRecordsGroupedByChannel = blocklist.groupBy(
(blocklistRecord: sdk.Models.BlocklistRecord) => blocklistRecord.channel
)
data.guidesGroupedByStreamId = guides
.filter((guide: sdk.Models.Guide) => !!guide.sources.length)
.groupBy((guide: sdk.Models.Guide) => guide.getStreamId())
data.categories = categories
data.countries = countries
data.subdivisions = subdivisions
Expand Down
6 changes: 5 additions & 1 deletion scripts/commands/playlist/format.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,8 @@ import { Storage } from '@freearhey/storage-js'
import { STREAMS_DIR } from '../../constants'
import { PlaylistParser } from '../../core'
import { getStreamInfo } from '../../utils'
import { loadData, data } from '../../api'
import cliProgress from 'cli-progress'
import { loadData } from '../../api'
import { eachLimit } from 'async'
import path from 'node:path'
import os from 'node:os'
Expand Down Expand Up @@ -44,6 +44,10 @@ async function main() {
let files = program.args.length ? program.args : await streamsStorage.list('**/*.m3u')
files = files.map((filepath: string) => path.basename(filepath))
let streams = await parser.parse(files)
streams = streams.map((stream: Stream) => {
stream.setGuides(data.guidesGroupedByStreamId.get(stream.getId()))
return stream
})

logger.info(`found ${streams.count()} streams`)

Expand Down
33 changes: 9 additions & 24 deletions scripts/commands/playlist/generate.ts
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,10 @@ async function main() {
})
const files = await streamsStorage.list('**/*.m3u')
let streams = await parser.parse(files)
streams = streams.map((stream: Stream) => {
stream.setGuides(data.guidesGroupedByStreamId.get(stream.getId()))
return stream
})
const totalStreams = streams.count()
logger.info(`found ${totalStreams} streams`)

Expand Down Expand Up @@ -62,32 +66,16 @@ async function main() {
await new LanguagesGenerator({ streams, logFile }).generate()

logger.info('generating countries/...')
await new CountriesGenerator({
countries,
streams,
logFile
}).generate()
await new CountriesGenerator({ countries, streams, logFile }).generate()

logger.info('generating subdivisions/...')
await new SubdivisionsGenerator({
subdivisions,
streams,
logFile
}).generate()
await new SubdivisionsGenerator({ subdivisions, streams, logFile }).generate()

logger.info('generating cities/...')
await new CitiesGenerator({
cities,
streams,
logFile
}).generate()
await new CitiesGenerator({ cities, streams, logFile }).generate()

logger.info('generating regions/...')
await new RegionsGenerator({
streams,
regions,
logFile
}).generate()
await new RegionsGenerator({ streams, regions, logFile }).generate()

logger.info('generating sources/...')
await new SourcesGenerator({ streams, logFile }).generate()
Expand All @@ -99,10 +87,7 @@ async function main() {
await new IndexCategoryGenerator({ streams, logFile }).generate()

logger.info('generating index.country.m3u...')
await new IndexCountryGenerator({
streams,
logFile
}).generate()
await new IndexCountryGenerator({ streams, logFile }).generate()

logger.info('generating index.language.m3u...')
await new IndexLanguageGenerator({ streams, logFile }).generate()
Expand Down
10 changes: 9 additions & 1 deletion scripts/commands/playlist/update.ts
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,10 @@ async function loadStreams() {
const files = await streamsStorage.list('**/*.m3u')

streams = await parser.parse(files)
streams = streams.map((stream: Stream) => {
stream.setGuides(apiData.guidesGroupedByStreamId.get(stream.getId()))
return stream
})
}

async function processIssues(issues: Collection<Issue>) {
Expand Down Expand Up @@ -132,7 +136,7 @@ async function removeStream(issue: Issue) {
if (changed) {
processedIssues.add(issue)
} else {
log.error(`None of the URLs specified in the request were found in the playlists`)
log.error('None of the URLs specified in the request were found in the playlists')
skippedIssues.add(issue)
}
}
Expand Down Expand Up @@ -162,6 +166,8 @@ async function editStream(issue: Issue) {

stream.updateWithIssue(data)

stream.setGuides(apiData.guidesGroupedByStreamId.get(stream.getId()))

const errors = new Collection<Error>()
errors.concat(stream.validate())
if (errors.isNotEmpty()) {
Expand Down Expand Up @@ -260,6 +266,8 @@ async function addStream(issue: Issue) {

stream.updateTitle().updateFilepath()

stream.setGuides(apiData.guidesGroupedByStreamId.get(stream.getId()))

streams.add(stream)

const errors = new Collection<Error>()
Expand Down
53 changes: 46 additions & 7 deletions scripts/models/playlist.ts
Original file line number Diff line number Diff line change
@@ -1,26 +1,65 @@
import { Collection } from '@freearhey/core'
import * as sdk from '@iptv-org/sdk'
import { Stream } from '../models'

type PlaylistOptions = {
public: boolean
public?: boolean
}

export class Playlist {
streams: Collection<Stream>
options: {
public: boolean
}
public: boolean

constructor(streams: Collection<Stream>, options?: PlaylistOptions) {
this.streams = streams
this.options = options || { public: false }
this.public = options?.public === true
}

getGuides(): Collection<sdk.Models.Guide> {
const guides = new Collection<sdk.Models.Guide>()

this.streams.forEach(stream => {
guides.concat(stream.getGuides())
})

return guides
}

getGuideUrls(): Collection<string> {
function byFormat(source: sdk.Types.GuideSource) {
if (source.format === 'GZIP') return 2
if (source.format === 'XML') return 1
return 0
}

const urls = new Collection<string>()

this.getGuides().forEach((guide: sdk.Models.Guide) => {
const sources = new Collection(guide.sources)

const source = sources
.filter((source: sdk.Types.GuideSource) => ['GZIP', 'XML'].includes(source.format))
.sortBy([byFormat], ['desc'])
.first()

if (source) urls.add(source.url)
})

return urls.uniq()
}

toString() {
let output = '#EXTM3U\r\n'
let output = '#EXTM3U'

const guideUrls = this.getGuideUrls()
if (guideUrls.count()) {
output += ` x-tvg-url="${guideUrls.join(',')}"`
}

output += '\r\n'

this.streams.forEach((stream: Stream) => {
output += stream.toString(this.options) + '\r\n'
output += stream.toString({ public: this.public }) + '\r\n'
})

return output
Expand Down
13 changes: 11 additions & 2 deletions scripts/models/stream.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,15 @@ export class Stream extends sdk.Models.Stream {
removed: boolean = false
tvgId?: string
statusCode?: string
guides = new Collection<sdk.Models.Guide>()

setGuides(guides?: sdk.Models.Guide[]) {
this.guides = new Collection(guides)
}

getGuides(): Collection<sdk.Models.Guide> {
return this.guides
}

validate(): Collection<Error> {
const errors = new Collection<Error>()
Expand Down Expand Up @@ -80,12 +89,12 @@ export class Stream extends sdk.Models.Stream {
quality: quality || null,
url: data.url,
referrer: data.http.referrer || null,
user_agent: data.http['user-agent'] || null
user_agent: data.http['user-agent'] || null,
label: label || null
})

stream.tvgId = data.tvg.id
stream.line = data.line
stream.label = label || null

return stream
}
Expand Down
2 changes: 1 addition & 1 deletion tests/__data__/expected/playlist_format/nl.m3u
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
#EXTM3U
#EXTM3U x-tvg-url="https://example.com/guide.xml.gz"
#EXTINF:-1 tvg-id="NPO1.nl@SD",NPO 1 (342p) [Geo-blocked]
http://resolver.streaming.api.nos.nl/livestream?url=/live/npo/tvlive/npo1/npo1.isml/.m3u8
#EXTINF:-1 tvg-id="NPO2.nl@SD",NPO 2 (1080p) [Geo-blocked]
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
#EXTM3U
#EXTM3U x-tvg-url="https://example2.com/guide.xml,https://example.com/guide.xml.gz"
#EXTINF:-1 tvg-id="5AABTV.ca" tvg-logo="" group-title="Undefined",5AAB TV
http://158.69.124.9:1935/5aabtv/5aabtv/playlist.m3u8
#EXTINF:-1 tvg-id="AlJazeera.qa@Arabic" tvg-logo="" group-title="Undefined",Al Jazeera (1080p)
Expand Down
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
#EXTM3U
#EXTM3U x-tvg-url="https://example.com/guide.xml.gz"
#EXTINF:-1 tvg-id="AndorraTV.ad@SD" tvg-logo="https://i.imgur.com/BnhTn8i.png" group-title="Undefined",ATV
https://iptv-all.lanesh4d0w.repl.co/andorra/atv
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
#EXTM3U
#EXTM3U x-tvg-url="https://example.com/guide.xml.gz"
#EXTINF:-1 tvg-id="AndorraTV.ad@SD" tvg-logo="https://i.imgur.com/BnhTn8i.png" group-title="Undefined",ATV
https://iptv-all.lanesh4d0w.repl.co/andorra/atv
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
#EXTM3U
#EXTM3U x-tvg-url="https://example2.com/guide.xml,https://example.com/guide.xml.gz"
#EXTINF:-1 tvg-id="AlJazeera.qa@Arabic" tvg-logo="" group-title="Undefined",Al Jazeera (1080p)
https://live-hls-apps-aja-fa.getaj.net/AJA/index.m3u8
#EXTINF:-1 tvg-id="" tvg-logo="" http-referrer="http://imn.iq" http-user-agent="Mozilla/5.0 (iPhone; CPU iPhone OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148" group-title="Undefined",Andorra TV (720p) [Not 24/7]
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
#EXTM3U
#EXTM3U x-tvg-url="https://example2.com/guide.xml,https://example.com/guide.xml.gz"
#EXTINF:-1 tvg-id="BBCNews.uk" tvg-logo="https://raw.githubusercontent.com/Tapiosinn/tv-logos/master/countries/united-kingdom/bbc-news-uk.png" group-title="General",BBC News HD
http://1111296894.rsc.cdn77.org/LS-ATL-54548-6/index.m3u8
#EXTINF:-1 tvg-id="LDPRTV.ru" tvg-logo="https://iptvx.one/icn/ldpr-tv.png" group-title="General",ЛДПР ТВ (1080p)
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
#EXTM3U
#EXTM3U x-tvg-url="https://example.com/guide.xml.gz,https://example2.com/guide.xml"
#EXTINF:-1 tvg-id="AndorraTV.ad@SD" tvg-logo="https://i.imgur.com/BnhTn8i.png" group-title="Andorra",ATV
https://iptv-all.lanesh4d0w.repl.co/andorra/atv
#EXTINF:-1 tvg-id="5AABTV.ca" tvg-logo="" group-title="Canada",5AAB TV
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
#EXTM3U
#EXTM3U x-tvg-url="https://example.com/guide.xml.gz,https://example2.com/guide.xml"
#EXTINF:-1 tvg-id="AndorraTV.ad@SD" tvg-logo="https://i.imgur.com/BnhTn8i.png" group-title="Catalan",ATV
https://iptv-all.lanesh4d0w.repl.co/andorra/atv
#EXTINF:-1 tvg-id="BBCNews.uk" tvg-logo="https://raw.githubusercontent.com/Tapiosinn/tv-logos/master/countries/united-kingdom/bbc-news-uk.png" group-title="English",BBC News HD
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
#EXTM3U
#EXTM3U x-tvg-url="https://example2.com/guide.xml,https://example.com/guide.xml.gz"
#EXTINF:-1 tvg-id="5AABTV.ca" tvg-logo="" group-title="Undefined",5AAB TV
http://158.69.124.9:1935/5aabtv/5aabtv/playlist.m3u8
#EXTINF:-1 tvg-id="AlJazeera.qa@Arabic" tvg-logo="" group-title="Undefined",Al Jazeera (1080p)
Expand Down
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
#EXTM3U
#EXTM3U x-tvg-url="https://example.com/guide.xml.gz"
#EXTINF:-1 tvg-id="AndorraTV.ad@SD" tvg-logo="https://i.imgur.com/BnhTn8i.png" group-title="Undefined",ATV
https://iptv-all.lanesh4d0w.repl.co/andorra/atv
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
#EXTM3U
#EXTM3U x-tvg-url="https://example2.com/guide.xml,https://example.com/guide.xml.gz"
#EXTINF:-1 tvg-id="5AABTV.ca" tvg-logo="" group-title="Undefined",5AAB TV
http://158.69.124.9:1935/5aabtv/5aabtv/playlist.m3u8
#EXTINF:-1 tvg-id="AlJazeera.qa@Arabic" tvg-logo="" group-title="Undefined",Al Jazeera (1080p)
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
#EXTM3U
#EXTM3U x-tvg-url="https://example.com/guide.xml.gz"
#EXTINF:-1 tvg-id="Zoo.ad@HD" tvg-logo="https://i.imgur.com/ciTJrnl.png" group-title="Undefined",Zoo (720p)
https://iptv-all.lanesh4d0w.repl.co/andorra/zoo
#EXTINF:-1 tvg-id="AndorraTV.ad@SD" tvg-logo="https://i.imgur.com/BnhTn8i.png" group-title="Undefined",ATV
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
#EXTM3U
#EXTM3U x-tvg-url="https://example.com/guide.xml.gz"
#EXTINF:-1 tvg-id="AndorraTV.ad@SD" tvg-logo="https://i.imgur.com/BnhTn8i.png" group-title="Undefined",ATV
https://iptv-all.lanesh4d0w.repl.co/andorra/atv
#EXTINF:-1 tvg-id="BBCNews.uk" tvg-logo="https://raw.githubusercontent.com/Tapiosinn/tv-logos/master/countries/united-kingdom/bbc-news-uk.png" group-title="General;News",BBC News HD
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
#EXTM3U
#EXTM3U x-tvg-url="https://example.com/guide.xml.gz"
#EXTINF:-1 tvg-id="AndorraTV.ad@SD" tvg-logo="https://i.imgur.com/BnhTn8i.png" group-title="Undefined",ATV
https://iptv-all.lanesh4d0w.repl.co/andorra/atv
#EXTINF:-1 tvg-id="BBCNews.uk" tvg-logo="https://raw.githubusercontent.com/Tapiosinn/tv-logos/master/countries/united-kingdom/bbc-news-uk.png" group-title="General;News",BBC News HD
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
#EXTM3U
#EXTM3U x-tvg-url="https://example.com/guide.xml.gz"
#EXTINF:-1 tvg-id="5AABTV.ca" tvg-logo="" group-title="Undefined",5AAB TV
http://158.69.124.9:1935/5aabtv/5aabtv/playlist.m3u8
#EXTINF:-1 tvg-id="AndorraTV.ad@SD" tvg-logo="https://i.imgur.com/BnhTn8i.png" group-title="Undefined",ATV
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
#EXTM3U
#EXTM3U x-tvg-url="https://example.com/guide.xml.gz"
#EXTINF:-1 tvg-id="AndorraTV.ad@HD" tvg-logo="https://i.imgur.com/CnhTn8i.png" group-title="Undefined",ATV HD
https://iptv-all.lanesh4d0w.repl.co/andorra/atv_hd
#EXTINF:-1 tvg-id="AndorraTV.ad@SD" tvg-logo="https://i.imgur.com/BnhTn8i.png" group-title="Undefined",ATV
Expand Down
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
#EXTM3U
#EXTM3U x-tvg-url="https://example.com/guide.xml.gz"
#EXTINF:-1 tvg-id="AndorraTV.ad@SD" tvg-logo="https://i.imgur.com/BnhTn8i.png" group-title="Undefined",ATV
https://iptv-all.lanesh4d0w.repl.co/andorra/atv
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
#EXTM3U
#EXTM3U x-tvg-url="https://example.com/guide.xml.gz"
#EXTINF:-1 tvg-id="AndorraTV.ad@SD" tvg-logo="https://i.imgur.com/BnhTn8i.png" group-title="Undefined",ATV
https://iptv-all.lanesh4d0w.repl.co/andorra/atv
2 changes: 1 addition & 1 deletion tests/__data__/expected/playlist_update/streams/fr.m3u
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
#EXTM3U
#EXTM3U x-tvg-url="https://example.com/guide.xml.gz"
#EXTINF:-1 tvg-id="TFX.fr@SD",TFX
#EXTVLCOPT:http-referrer=https://pkpakiplay.xyz/
#EXTVLCOPT:http-user-agent=Mozilla/5.0 (iPhone; CPU iPhone OS 17_7 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.0 Mobile/15E148 Safari/604.1
Expand Down
8 changes: 8 additions & 0 deletions tests/__data__/input/data/channels.json
Original file line number Diff line number Diff line change
Expand Up @@ -596,5 +596,13 @@
"country": "BZ",
"categories": [],
"is_nsfw": false
},
{
"id": "Tele2000.br",
"name": "Tele 2000",
"network": null,
"country": "BR",
"categories": [],
"is_nsfw": false
}
]
Loading