Update EPG model and parsing mechanisms
This commit revises the EPG model's struct tag to include mapstructure along with json. It also includes an implementation switch from goquery and otto to mapstructure for data parsing. This removes dependencies on PuerkitoBio/goquery and robertkrimen/otto. The commit also includes some minor code refactors for the more efficient handling of EPG data along with a version bump to 0.0.3.
This commit is contained in:
Senis
parent
8b3020e086
commit
fe53df21b7
@ -4,6 +4,7 @@ RUN go env -w GOPROXY=https://goproxy.cn,direct
|
||||
|
||||
WORKDIR /build
|
||||
COPY . .
|
||||
RUN go mod tidy
|
||||
RUN go mod download
|
||||
RUN go build -ldflags="-s -w" -trimpath -v -o /app/main main.go
|
||||
|
||||
|
||||
@ -1,33 +1,16 @@
|
||||
package channel
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/PuerkitoBio/goquery"
|
||||
"github.com/mitchellh/mapstructure"
|
||||
)
|
||||
|
||||
func BytesToChannels(resp []byte) ([]Channel, error) {
|
||||
doc, err := goquery.NewDocumentFromReader(bytes.NewBuffer(resp))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var data []string
|
||||
re := regexp.MustCompile(`\((.*?)\)`) // filter strings in ()
|
||||
re1 := regexp.MustCompile(`'([^']*)'`) // filter strings in ''
|
||||
|
||||
// use the goquery document...
|
||||
_ = doc.Find("script:contains(ChannelName)").Each(func(i int, selection *goquery.Selection) {
|
||||
matches := re.FindStringSubmatch(selection.Text())
|
||||
if len(matches) > 1 {
|
||||
matches2 := re1.FindAllStringSubmatch(matches[1], -1)
|
||||
data = append(data, matches2[1][1]) // 用1而非0作为索引
|
||||
}
|
||||
})
|
||||
re := regexp.MustCompile(`(?s)ChannelID="\d*".*?ChannelFECPort="\d*"`)
|
||||
data := re.FindAllString(string(resp), -1)
|
||||
|
||||
var channelMaps []map[string]any
|
||||
re2 := regexp.MustCompile(`画中画|单音轨`)
|
||||
@ -36,7 +19,8 @@ func BytesToChannels(resp []byte) ([]Channel, error) {
|
||||
continue
|
||||
}
|
||||
|
||||
res := strings.Split(data[i], ",")
|
||||
d := data[i]
|
||||
res := strings.Split(d, ",")
|
||||
kvMap := make(map[string]any)
|
||||
for ii := range res {
|
||||
kvs := strings.SplitN(res[ii], "=", 2)
|
||||
|
||||
@ -1,87 +1,87 @@
|
||||
package epg
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"time"
|
||||
|
||||
"github.com/PuerkitoBio/goquery"
|
||||
"github.com/robertkrimen/otto"
|
||||
"github.com/mitchellh/mapstructure"
|
||||
)
|
||||
|
||||
func byteToEpg(resp []byte) ([]Epg, error) {
|
||||
doc, err := goquery.NewDocumentFromReader(bytes.NewReader(resp))
|
||||
if err != nil {
|
||||
func BytesToAllEPGs(resp []byte) ([]Epg, error) {
|
||||
re := regexp.MustCompile(`(?s)\[.*]`)
|
||||
b := re.FindSubmatch(resp)
|
||||
if b == nil {
|
||||
return nil, errors.New("not found valid data")
|
||||
}
|
||||
|
||||
var es []any
|
||||
if err := json.Unmarshal(b[0], &es); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
vm := otto.New()
|
||||
if err := vm.Set("parent", map[string]any{}); err != nil {
|
||||
return nil, err
|
||||
if len(es) != 2 {
|
||||
return nil, errors.New("the length of data must equal 2")
|
||||
}
|
||||
|
||||
if _, err := vm.Run(doc.Find("script").Text()); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
objParent, _ := vm.Object("parent")
|
||||
obj, _ := objParent.Get("jsonBackLookStr")
|
||||
|
||||
ex, _ := obj.Export()
|
||||
val, ok := ex.([]any)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("invalid type of object")
|
||||
}
|
||||
if len(val) < 2 {
|
||||
return nil, errors.New("invalid length of object")
|
||||
}
|
||||
es := val[1]
|
||||
|
||||
bb, _ := json.Marshal(es)
|
||||
var data [][]Epg
|
||||
if err := json.Unmarshal(bb, &data); err != nil {
|
||||
if err := mapstructure.Decode(es[1], &data); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var epgs []Epg
|
||||
for i := range data {
|
||||
epgsDay := data[i]
|
||||
for ii := range epgsDay {
|
||||
epgs = append(epgs, epgsDay[ii])
|
||||
e := data[i]
|
||||
for ii := range e {
|
||||
epgs = append(epgs, e[ii])
|
||||
}
|
||||
}
|
||||
|
||||
return epgs, nil
|
||||
}
|
||||
|
||||
func GetEPGs(resp []byte) ([]Epg, error) {
|
||||
epgs, err := byteToEpg(resp)
|
||||
func BytesToValidEPGs(resp []byte) ([]Epg, error) {
|
||||
allEPGs, err := BytesToAllEPGs(resp)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var validEpgs []Epg
|
||||
var epgs []Epg
|
||||
tz := time.FixedZone("CST", 3600*8)
|
||||
for i := range epgs {
|
||||
endTime, err := time.ParseInLocation("20060102150405", epgs[i].EndTimeFormat, tz) // time format: 20231228001700
|
||||
for i := range allEPGs {
|
||||
epg, err := filterEPGs(&allEPGs[i], tz)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
beginTime, err := time.ParseInLocation("20060102150405", epgs[i].BeginTimeFormat, tz) // time format: 20231228001700
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
if beginTime.Sub(endTime) > 0 {
|
||||
endTime = beginTime
|
||||
}
|
||||
|
||||
if time.Since(endTime) < time.Hour {
|
||||
validEpgs = append(validEpgs, epgs[i])
|
||||
}
|
||||
epgs = append(epgs, *epg)
|
||||
}
|
||||
return validEpgs, nil
|
||||
|
||||
return epgs, nil
|
||||
}
|
||||
|
||||
func filterEPGs(e *Epg, tz *time.Location) (*Epg, error) {
|
||||
// time format: 20231228001700
|
||||
endTime, err := time.ParseInLocation("20060102150405", e.EndTimeFormat, tz)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
beginTime, err := time.ParseInLocation("20060102150405", e.BeginTimeFormat, tz)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if beginTime.Sub(endTime) > 0 {
|
||||
endTime = beginTime
|
||||
}
|
||||
|
||||
if time.Since(endTime) > time.Hour {
|
||||
return nil, fmt.Errorf("not a valid EPG: %s [%s] -> %s", e.ChannelId, e.ProgramName, e.EndTimeFormat)
|
||||
|
||||
}
|
||||
|
||||
return e, nil
|
||||
}
|
||||
|
||||
@ -15,19 +15,21 @@ func TestGetEpg(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
es, err := byteToEpg(resp)
|
||||
es, err := BytesToAllEPGs(resp)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
for i := range es {
|
||||
t.Log(es[i])
|
||||
}
|
||||
t.Logf("Total EPGs: %d", len(es))
|
||||
|
||||
validEs, err := GetEPGs(resp)
|
||||
validEs, err := BytesToValidEPGs(resp)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
for i := range validEs {
|
||||
t.Log(validEs[i])
|
||||
}
|
||||
t.Logf("Valid EPGs: %d", len(validEs))
|
||||
}
|
||||
|
||||
@ -1,13 +1,13 @@
|
||||
package epg
|
||||
|
||||
type Epg struct {
|
||||
BeginTimeFormat string `json:"beginTimeFormat"`
|
||||
ChannelId string `json:"channelId"`
|
||||
ContentId string `json:"contentId"`
|
||||
EndTime string `json:"endTime"`
|
||||
EndTimeFormat string `json:"endTimeFormat"`
|
||||
Index string `json:"index"`
|
||||
IsPlayable string `json:"isPlayable"`
|
||||
ProgramName string `json:"programName"`
|
||||
StartTime string `json:"startTime"`
|
||||
BeginTimeFormat string `json:"beginTimeFormat" mapstructure:"beginTimeFormat"`
|
||||
ChannelId string `json:"channelId" mapstructure:"channelId"`
|
||||
ContentId string `json:"contentId" mapstructure:"contentId"`
|
||||
EndTime string `json:"endTime" mapstructure:"endTime"`
|
||||
EndTimeFormat string `json:"endTimeFormat" mapstructure:"endTimeFormat"`
|
||||
Index string `json:"index" mapstructure:"index"`
|
||||
IsPlayable string `json:"isPlayable" mapstructure:"isPlayable"`
|
||||
ProgramName string `json:"programName" mapstructure:"programName"`
|
||||
StartTime string `json:"startTime" mapstructure:"startTime"`
|
||||
}
|
||||
|
||||
@ -6,7 +6,7 @@ import (
|
||||
|
||||
const (
|
||||
appName = "iptvChannel"
|
||||
version = "0.0.2"
|
||||
version = "0.0.3"
|
||||
desc = "(github.com/thank243/iptvChannel)"
|
||||
)
|
||||
|
||||
|
||||
@ -121,15 +121,13 @@ func (c *Controller) fetchEPGs() error {
|
||||
logger.Error(err)
|
||||
return
|
||||
}
|
||||
epgs, err := epg.GetEPGs(resp)
|
||||
epgs, err := epg.BytesToValidEPGs(resp)
|
||||
if err != nil {
|
||||
logger.Error(err)
|
||||
return
|
||||
}
|
||||
if len(epgs) > 0 {
|
||||
for i := range epgs {
|
||||
es <- epgs[i]
|
||||
}
|
||||
for i := range epgs {
|
||||
es <- epgs[i]
|
||||
}
|
||||
}(i)
|
||||
}
|
||||
|
||||
6
go.mod
6
go.mod
@ -3,23 +3,20 @@ module github.com/thank243/iptvChannel
|
||||
go 1.21
|
||||
|
||||
require (
|
||||
github.com/PuerkitoBio/goquery v1.8.1
|
||||
github.com/beevik/etree v1.3.0
|
||||
github.com/go-resty/resty/v2 v2.11.0
|
||||
github.com/labstack/echo/v4 v4.11.4
|
||||
github.com/labstack/gommon v0.4.2
|
||||
github.com/mitchellh/mapstructure v1.5.0
|
||||
github.com/robertkrimen/otto v0.3.0
|
||||
github.com/robfig/cron/v3 v3.0.1
|
||||
github.com/sirupsen/logrus v1.9.3
|
||||
github.com/spf13/viper v1.18.2
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/andybalholm/cascadia v1.3.1 // indirect
|
||||
github.com/fsnotify/fsnotify v1.7.0 // indirect
|
||||
github.com/golang-jwt/jwt v3.2.2+incompatible // indirect
|
||||
github.com/hashicorp/hcl v1.0.0 // indirect
|
||||
github.com/labstack/gommon v0.4.2 // indirect
|
||||
github.com/magiconair/properties v1.8.7 // indirect
|
||||
github.com/mattn/go-colorable v0.1.13 // indirect
|
||||
github.com/mattn/go-isatty v0.0.20 // indirect
|
||||
@ -42,6 +39,5 @@ require (
|
||||
golang.org/x/text v0.14.0 // indirect
|
||||
golang.org/x/time v0.5.0 // indirect
|
||||
gopkg.in/ini.v1 v1.67.0 // indirect
|
||||
gopkg.in/sourcemap.v1 v1.0.5 // indirect
|
||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||
)
|
||||
|
||||
12
go.sum
12
go.sum
@ -1,7 +1,3 @@
|
||||
github.com/PuerkitoBio/goquery v1.8.1 h1:uQxhNlArOIdbrH1tr0UXwdVFgDcZDrZVdcpygAcwmWM=
|
||||
github.com/PuerkitoBio/goquery v1.8.1/go.mod h1:Q8ICL1kNUJ2sXGoAhPGUdYDJvgQgHzJsnnd3H7Ho5jQ=
|
||||
github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c=
|
||||
github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA=
|
||||
github.com/beevik/etree v1.3.0 h1:hQTc+pylzIKDb23yYprodCWWTt+ojFfUZyzU09a/hmU=
|
||||
github.com/beevik/etree v1.3.0/go.mod h1:aiPf89g/1k3AShMVAzriilpcE4R/Vuor90y83zVZWFc=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
@ -42,8 +38,6 @@ github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdU
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
|
||||
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/robertkrimen/otto v0.3.0 h1:5RI+8860NSxvXywDY9ddF5HcPw0puRsd8EgbXV0oqRE=
|
||||
github.com/robertkrimen/otto v0.3.0/go.mod h1:uW9yN1CYflmUQYvAMS0m+ZiNo3dMzRUDQJX0jWbzgxw=
|
||||
github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=
|
||||
github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
|
||||
github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=
|
||||
@ -95,10 +89,8 @@ golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91
|
||||
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
|
||||
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
|
||||
golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
|
||||
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
|
||||
golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
|
||||
golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
|
||||
golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
|
||||
golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c=
|
||||
@ -108,7 +100,6 @@ golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJ
|
||||
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
@ -127,7 +118,6 @@ golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
|
||||
golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
|
||||
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
|
||||
golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
|
||||
@ -147,8 +137,6 @@ gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogR
|
||||
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA=
|
||||
gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
|
||||
gopkg.in/sourcemap.v1 v1.0.5 h1:inv58fC9f9J3TK2Y2R1NPntXEn3/wjWHkonhIUODNTI=
|
||||
gopkg.in/sourcemap.v1 v1.0.5/go.mod h1:2RlvNNSMglmRrcvhfuzp4hQHwOtjxlbjX7UPY/GXb78=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
|
||||
Loading…
Reference in New Issue
Block a user