Module refactoring (#2)
* Move main to cmd * Add search module * Clean-ups on search module * Add modules xdcc, pb, table and util * Update READMEmain
parent
1162bdb9fa
commit
ed410faf0a
|
|
@ -22,20 +22,20 @@ Assuming you have the latest version of Go installed on your system, you can use
|
||||||
```bash
|
```bash
|
||||||
git clone https://github.com/ostafen/xdcc-cli.git
|
git clone https://github.com/ostafen/xdcc-cli.git
|
||||||
cd xdcc-cli
|
cd xdcc-cli
|
||||||
go build -o xdcc .
|
go build -o xdcc-cli cmd/main.go
|
||||||
```
|
```
|
||||||
|
|
||||||
## Usage
|
## Usage
|
||||||
To initialize a file search, simply pass a list of keywords to the **search** subcommand like so:
|
To initialize a file search, simply pass a list of keywords to the **search** subcommand like so:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
foo@bar:~$ xdcc search keyword1 keyword2 ...
|
foo@bar:~$ xdcc-cli search keyword1 keyword2 ...
|
||||||
```
|
```
|
||||||
|
|
||||||
For example, to search for the latest iso of ubuntu, you could simply write:
|
For example, to search for the latest iso of ubuntu, you could simply write:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
foo@bar:~$ xdcc search ubuntu iso
|
foo@bar:~$ xdcc-cli search ubuntu iso
|
||||||
```
|
```
|
||||||
|
|
||||||
If the command succedeeds, a table, similar to the following, will be displayed:
|
If the command succedeeds, a table, similar to the following, will be displayed:
|
||||||
|
|
@ -49,7 +49,7 @@ A part from file details, each row will contain an **url** of the form irc://net
|
||||||
To download one or more file, simply pass a list of url to the **get** subcommand like so:
|
To download one or more file, simply pass a list of url to the **get** subcommand like so:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
foo@bar:~$ xdcc get url1 url2 ... [-o /path/to/an/output/directory]
|
foo@bar:~$ xdcc-cli get url1 url2 ... [-o /path/to/an/output/directory]
|
||||||
```
|
```
|
||||||
Alternatively, you could also specify a .txt input file, containing a list of urls (one for each line), using the **-i** switch.
|
Alternatively, you could also specify a .txt input file, containing a list of urls (one for each line), using the **-i** switch.
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -9,24 +9,23 @@ import (
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
|
"xdcc-cli/pb"
|
||||||
|
"xdcc-cli/search"
|
||||||
|
table "xdcc-cli/table"
|
||||||
|
xdcc "xdcc-cli/xdcc"
|
||||||
)
|
)
|
||||||
|
|
||||||
var registry *XdccProviderRegistry = nil
|
var searchEngine *search.ProviderAggregator
|
||||||
|
|
||||||
func init() {
|
func init() {
|
||||||
registry = NewProviderRegistry()
|
searchEngine = search.NewProviderAggregator(
|
||||||
registry.AddProvider(&XdccEuProvider{})
|
&search.XdccEuProvider{},
|
||||||
registry.AddProvider(&SunXdccProvider{})
|
&search.SunXdccProvider{},
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
var defaultColWidths []int = []int{100, 10, -1}
|
var defaultColWidths []int = []int{100, 10, -1}
|
||||||
|
|
||||||
const (
|
|
||||||
KiloByte = 1024
|
|
||||||
MegaByte = KiloByte * 1024
|
|
||||||
GigaByte = MegaByte * 1024
|
|
||||||
)
|
|
||||||
|
|
||||||
func FloatToString(value float64) string {
|
func FloatToString(value float64) string {
|
||||||
if value-float64(int64(value)) > 0 {
|
if value-float64(int64(value)) > 0 {
|
||||||
return strconv.FormatFloat(value, 'f', 2, 32)
|
return strconv.FormatFloat(value, 'f', 2, 32)
|
||||||
|
|
@ -39,12 +38,12 @@ func formatSize(size int64) string {
|
||||||
return "--"
|
return "--"
|
||||||
}
|
}
|
||||||
|
|
||||||
if size >= GigaByte {
|
if size >= search.GigaByte {
|
||||||
return FloatToString(float64(size)/float64(GigaByte)) + "GB"
|
return FloatToString(float64(size)/float64(search.GigaByte)) + "GB"
|
||||||
} else if size >= MegaByte {
|
} else if size >= search.MegaByte {
|
||||||
return FloatToString(float64(size)/float64(MegaByte)) + "MB"
|
return FloatToString(float64(size)/float64(search.MegaByte)) + "MB"
|
||||||
} else if size >= KiloByte {
|
} else if size >= search.KiloByte {
|
||||||
return FloatToString(float64(size)/float64(KiloByte)) + "KB"
|
return FloatToString(float64(size)/float64(search.KiloByte)) + "KB"
|
||||||
}
|
}
|
||||||
return FloatToString(float64(size)) + "B"
|
return FloatToString(float64(size)) + "B"
|
||||||
}
|
}
|
||||||
|
|
@ -55,7 +54,7 @@ func searchCommand(args []string) {
|
||||||
|
|
||||||
args = parseFlags(searchCmd, args)
|
args = parseFlags(searchCmd, args)
|
||||||
|
|
||||||
printer := NewTablePrinter([]string{"File Name", "Size", "URL"})
|
printer := table.NewTablePrinter([]string{"File Name", "Size", "URL"})
|
||||||
printer.SetMaxWidths(defaultColWidths)
|
printer.SetMaxWidths(defaultColWidths)
|
||||||
|
|
||||||
if len(args) < 1 {
|
if len(args) < 1 {
|
||||||
|
|
@ -63,9 +62,9 @@ func searchCommand(args []string) {
|
||||||
os.Exit(1)
|
os.Exit(1)
|
||||||
}
|
}
|
||||||
|
|
||||||
res, _ := registry.Search(args)
|
res, _ := searchEngine.Search(args)
|
||||||
for _, fileInfo := range res {
|
for _, fileInfo := range res {
|
||||||
printer.AddRow(Row{fileInfo.Name, formatSize(fileInfo.Size), fileInfo.URL.String()})
|
printer.AddRow(table.Row{fileInfo.Name, formatSize(fileInfo.Size), fileInfo.URL.String()})
|
||||||
}
|
}
|
||||||
|
|
||||||
sortColumn := 2
|
sortColumn := 2
|
||||||
|
|
@ -77,22 +76,22 @@ func searchCommand(args []string) {
|
||||||
printer.Print()
|
printer.Print()
|
||||||
}
|
}
|
||||||
|
|
||||||
func transferLoop(transfer *XdccTransfer) {
|
func transferLoop(transfer *xdcc.XdccTransfer) {
|
||||||
pb := NewProgressBar()
|
bar := pb.NewProgressBar()
|
||||||
|
|
||||||
evts := transfer.PollEvents()
|
evts := transfer.PollEvents()
|
||||||
quit := false
|
quit := false
|
||||||
for !quit {
|
for !quit {
|
||||||
e := <-evts
|
e := <-evts
|
||||||
switch evtType := e.(type) {
|
switch evtType := e.(type) {
|
||||||
case *TransferStartedEvent:
|
case *xdcc.TransferStartedEvent:
|
||||||
pb.SetTotal(int(evtType.FileSize))
|
bar.SetTotal(int(evtType.FileSize))
|
||||||
pb.SetFileName(evtType.FileName)
|
bar.SetFileName(evtType.FileName)
|
||||||
pb.SetState(ProgressStateDownloading)
|
bar.SetState(pb.ProgressStateDownloading)
|
||||||
case *TransferProgessEvent:
|
case *xdcc.TransferProgessEvent:
|
||||||
pb.Increment(int(evtType.transferBytes))
|
bar.Increment(int(evtType.TransferBytes))
|
||||||
case *TransferCompletedEvent:
|
case *xdcc.TransferCompletedEvent:
|
||||||
pb.SetState(ProgressStateCompleted)
|
bar.SetState(pb.ProgressStateCompleted)
|
||||||
quit = true
|
quit = true
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -105,7 +104,7 @@ func suggestUnknownAuthoritySwitch(err error) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func doTransfer(transfer *XdccTransfer) {
|
func doTransfer(transfer *xdcc.XdccTransfer) {
|
||||||
err := transfer.Start()
|
err := transfer.Start()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
fmt.Println(err)
|
fmt.Println(err)
|
||||||
|
|
@ -182,7 +181,7 @@ func getCommand(args []string) {
|
||||||
wg := sync.WaitGroup{}
|
wg := sync.WaitGroup{}
|
||||||
for _, urlStr := range urlList {
|
for _, urlStr := range urlList {
|
||||||
if strings.HasPrefix(urlStr, "irc://") {
|
if strings.HasPrefix(urlStr, "irc://") {
|
||||||
url, err := parseURL(urlStr)
|
url, err := xdcc.ParseURL(urlStr)
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
fmt.Println(err.Error())
|
fmt.Println(err.Error())
|
||||||
|
|
@ -190,8 +189,8 @@ func getCommand(args []string) {
|
||||||
}
|
}
|
||||||
|
|
||||||
wg.Add(1)
|
wg.Add(1)
|
||||||
transfer := NewXdccTransfer(*url, *path, !*noSSL, *skipCertificateCheck)
|
transfer := xdcc.NewTransfer(*url, *path, !*noSSL, *skipCertificateCheck)
|
||||||
go func(transfer *XdccTransfer) {
|
go func(transfer *xdcc.XdccTransfer) {
|
||||||
doTransfer(transfer)
|
doTransfer(transfer)
|
||||||
wg.Done()
|
wg.Done()
|
||||||
}(transfer)
|
}(transfer)
|
||||||
|
|
@ -1,7 +1,8 @@
|
||||||
package main
|
package pb
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"time"
|
"time"
|
||||||
|
"xdcc-cli/util"
|
||||||
|
|
||||||
"github.com/vbauerster/mpb/v7"
|
"github.com/vbauerster/mpb/v7"
|
||||||
"github.com/vbauerster/mpb/v7/decor"
|
"github.com/vbauerster/mpb/v7/decor"
|
||||||
|
|
@ -38,7 +39,7 @@ const (
|
||||||
)
|
)
|
||||||
|
|
||||||
func createMpbBar(p *mpb.Progress, total int, taskName string, state ProgressState, queueBar *mpb.Bar) *mpb.Bar {
|
func createMpbBar(p *mpb.Progress, total int, taskName string, state ProgressState, queueBar *mpb.Bar) *mpb.Bar {
|
||||||
displayName := cutStr(taskName, barMaxFileNameWidth)
|
displayName := util.CutStr(taskName, barMaxFileNameWidth)
|
||||||
|
|
||||||
len := len(displayName)
|
len := len(displayName)
|
||||||
if len != 0 {
|
if len != 0 {
|
||||||
284
search.go
284
search.go
|
|
@ -1,284 +0,0 @@
|
||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"encoding/json"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"log"
|
|
||||||
"net/http"
|
|
||||||
"strconv"
|
|
||||||
"strings"
|
|
||||||
"sync"
|
|
||||||
|
|
||||||
"github.com/PuerkitoBio/goquery"
|
|
||||||
)
|
|
||||||
|
|
||||||
type XdccFileInfo struct {
|
|
||||||
URL IRCFile
|
|
||||||
Name string
|
|
||||||
Size int64
|
|
||||||
Slot int
|
|
||||||
}
|
|
||||||
|
|
||||||
type XdccSearchProvider interface {
|
|
||||||
Search(keywords []string) ([]XdccFileInfo, error)
|
|
||||||
}
|
|
||||||
|
|
||||||
type XdccProviderRegistry struct {
|
|
||||||
providerList []XdccSearchProvider
|
|
||||||
}
|
|
||||||
|
|
||||||
const MaxProviders = 100
|
|
||||||
|
|
||||||
func NewProviderRegistry() *XdccProviderRegistry {
|
|
||||||
return &XdccProviderRegistry{
|
|
||||||
providerList: make([]XdccSearchProvider, 0, MaxProviders),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (registry *XdccProviderRegistry) AddProvider(provider XdccSearchProvider) {
|
|
||||||
registry.providerList = append(registry.providerList, provider)
|
|
||||||
}
|
|
||||||
|
|
||||||
const MaxResults = 1024
|
|
||||||
|
|
||||||
func (registry *XdccProviderRegistry) Search(keywords []string) ([]XdccFileInfo, error) {
|
|
||||||
allResults := make(map[IRCFile]XdccFileInfo)
|
|
||||||
|
|
||||||
mtx := sync.Mutex{}
|
|
||||||
|
|
||||||
wg := sync.WaitGroup{}
|
|
||||||
wg.Add(len(registry.providerList))
|
|
||||||
for _, p := range registry.providerList {
|
|
||||||
go func(p XdccSearchProvider) {
|
|
||||||
resList, err := p.Search(keywords)
|
|
||||||
if err != nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
mtx.Lock()
|
|
||||||
for _, res := range resList {
|
|
||||||
allResults[res.URL] = res
|
|
||||||
}
|
|
||||||
mtx.Unlock()
|
|
||||||
|
|
||||||
wg.Done()
|
|
||||||
}(p)
|
|
||||||
}
|
|
||||||
wg.Wait()
|
|
||||||
|
|
||||||
results := make([]XdccFileInfo, 0, MaxResults)
|
|
||||||
for _, res := range allResults {
|
|
||||||
results = append(results, res)
|
|
||||||
}
|
|
||||||
return results, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func parseFileSize(sizeStr string) (int64, error) {
|
|
||||||
if len(sizeStr) == 0 {
|
|
||||||
return -1, errors.New("empty string")
|
|
||||||
}
|
|
||||||
lastChar := sizeStr[len(sizeStr)-1]
|
|
||||||
sizePart := sizeStr[:len(sizeStr)-1]
|
|
||||||
|
|
||||||
size, err := strconv.ParseFloat(sizePart, 32)
|
|
||||||
|
|
||||||
if err != nil {
|
|
||||||
return -1, err
|
|
||||||
}
|
|
||||||
switch lastChar {
|
|
||||||
case 'G':
|
|
||||||
return int64(size * GigaByte), nil
|
|
||||||
case 'M':
|
|
||||||
return int64(size * MegaByte), nil
|
|
||||||
case 'K':
|
|
||||||
return int64(size * KiloByte), nil
|
|
||||||
}
|
|
||||||
return -1, errors.New("unable to parse: " + sizeStr)
|
|
||||||
}
|
|
||||||
|
|
||||||
type XdccEuProvider struct{}
|
|
||||||
|
|
||||||
const XdccEuURL = "https://www.xdcc.eu/search.php"
|
|
||||||
|
|
||||||
const xdccEuNumberOfEntries = 7
|
|
||||||
|
|
||||||
func (p *XdccEuProvider) parseFields(fields []string) (*XdccFileInfo, error) {
|
|
||||||
if len(fields) != xdccEuNumberOfEntries {
|
|
||||||
return nil, errors.New("unexpected number of search entry fields")
|
|
||||||
}
|
|
||||||
|
|
||||||
fInfo := &XdccFileInfo{}
|
|
||||||
fInfo.URL.Network = fields[0]
|
|
||||||
fInfo.URL.Channel = fields[1]
|
|
||||||
fInfo.URL.UserName = fields[2]
|
|
||||||
slot, err := strconv.Atoi(fields[3][1:])
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
fInfo.Size, _ = parseFileSize(fields[5]) // ignoring error
|
|
||||||
|
|
||||||
fInfo.Name = fields[6]
|
|
||||||
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
fInfo.Slot = slot
|
|
||||||
return fInfo, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (p *XdccEuProvider) Search(keywords []string) ([]XdccFileInfo, error) {
|
|
||||||
keywordString := strings.Join(keywords, " ")
|
|
||||||
searchkey := strings.Join(strings.Fields(keywordString), "+")
|
|
||||||
res, err := http.Get(XdccEuURL + "?searchkey=" + searchkey)
|
|
||||||
|
|
||||||
if err != nil {
|
|
||||||
log.Fatal(err)
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
defer res.Body.Close()
|
|
||||||
if res.StatusCode != http.StatusOK {
|
|
||||||
return nil, fmt.Errorf("status code error: %d %s", res.StatusCode, res.Status)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Load the HTML document
|
|
||||||
doc, err := goquery.NewDocumentFromReader(res.Body)
|
|
||||||
if err != nil {
|
|
||||||
log.Fatal(err)
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
fileInfos := make([]XdccFileInfo, 0)
|
|
||||||
doc.Find("tr").Each(func(_ int, s *goquery.Selection) {
|
|
||||||
fields := make([]string, 0)
|
|
||||||
|
|
||||||
var urlStr string
|
|
||||||
s.Children().Each(func(i int, si *goquery.Selection) {
|
|
||||||
if i == 1 {
|
|
||||||
value, exists := si.Find("a").First().Attr("href")
|
|
||||||
if exists {
|
|
||||||
urlStr = value
|
|
||||||
}
|
|
||||||
}
|
|
||||||
fields = append(fields, strings.TrimSpace(si.Text()))
|
|
||||||
})
|
|
||||||
|
|
||||||
info, err := p.parseFields(fields)
|
|
||||||
if err == nil {
|
|
||||||
url, err := parseURL(urlStr + "/" + info.URL.UserName + "/" + strconv.Itoa(info.Slot))
|
|
||||||
if err == nil {
|
|
||||||
info.URL = *url
|
|
||||||
fileInfos = append(fileInfos, *info)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
})
|
|
||||||
return fileInfos, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
const (
|
|
||||||
SunXdccURL = "http://sunxdcc.com/deliver.php"
|
|
||||||
SunXdccNumberOfEntries = 8
|
|
||||||
)
|
|
||||||
|
|
||||||
type SunXdccProvider struct{}
|
|
||||||
|
|
||||||
func (p *SunXdccProvider) parseFields(entry *SunXdccEntry, index int) (*XdccFileInfo, error) {
|
|
||||||
info := &XdccFileInfo{}
|
|
||||||
info.URL.Network = entry.Network[index]
|
|
||||||
info.URL.UserName = entry.Bot[index]
|
|
||||||
info.URL.Channel = entry.Channel[index]
|
|
||||||
|
|
||||||
slot, err := strconv.Atoi(entry.Packnum[index][1:])
|
|
||||||
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
sizeString := strings.TrimLeft(strings.TrimRight(entry.Fsize[index], "]"), "[")
|
|
||||||
|
|
||||||
info.Size, _ = parseFileSize(sizeString) // ignoring error
|
|
||||||
info.Name = entry.Fname[index]
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
info.Slot = slot
|
|
||||||
return info, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
type SunXdccEntry struct {
|
|
||||||
Botrec []string
|
|
||||||
Network []string
|
|
||||||
Bot []string
|
|
||||||
Channel []string
|
|
||||||
Packnum []string
|
|
||||||
Gets []string
|
|
||||||
Fsize []string
|
|
||||||
Fname []string
|
|
||||||
}
|
|
||||||
|
|
||||||
func (p *SunXdccProvider) Search(keywords []string) ([]XdccFileInfo, error) {
|
|
||||||
keywordString := strings.Join(keywords, " ")
|
|
||||||
searchkey := strings.Join(strings.Fields(keywordString), "+")
|
|
||||||
// see https://sunxdcc.com/#api for API definition
|
|
||||||
res, err := http.Get(SunXdccURL + "?sterm=" + searchkey)
|
|
||||||
|
|
||||||
if err != nil {
|
|
||||||
log.Fatal(err)
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
defer res.Body.Close()
|
|
||||||
if res.StatusCode != http.StatusOK {
|
|
||||||
return nil, fmt.Errorf("status code error: %d %s", res.StatusCode, res.Status)
|
|
||||||
}
|
|
||||||
|
|
||||||
entry, err := p.parseResponse(res)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
if !p.validateResult(entry) {
|
|
||||||
return nil, fmt.Errorf("Parse Error, not all fields have the same size")
|
|
||||||
}
|
|
||||||
|
|
||||||
fileInfos := make([]XdccFileInfo, 0)
|
|
||||||
for i := 0; i < len(entry.Botrec); i++ {
|
|
||||||
info, err := p.parseFields(entry, i)
|
|
||||||
if err == nil {
|
|
||||||
fileInfos = append(fileInfos, *info)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return fileInfos, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (*SunXdccProvider) validateResult(entry *SunXdccEntry) bool {
|
|
||||||
sizes := [8]int{
|
|
||||||
len(entry.Botrec),
|
|
||||||
len(entry.Network),
|
|
||||||
len(entry.Bot),
|
|
||||||
len(entry.Channel),
|
|
||||||
len(entry.Packnum),
|
|
||||||
len(entry.Gets),
|
|
||||||
len(entry.Fsize),
|
|
||||||
len(entry.Fname),
|
|
||||||
}
|
|
||||||
|
|
||||||
length := sizes[0]
|
|
||||||
for _, l := range sizes {
|
|
||||||
if length != l {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
|
|
||||||
func (*SunXdccProvider) parseResponse(res *http.Response) (*SunXdccEntry, error) {
|
|
||||||
entry := &SunXdccEntry{}
|
|
||||||
decoder := json.NewDecoder(res.Body)
|
|
||||||
err := decoder.Decode(entry)
|
|
||||||
return entry, err
|
|
||||||
}
|
|
||||||
|
|
@ -0,0 +1,98 @@
|
||||||
|
package search
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"strconv"
|
||||||
|
"sync"
|
||||||
|
"xdcc-cli/xdcc"
|
||||||
|
)
|
||||||
|
|
||||||
|
type XdccFileInfo struct {
|
||||||
|
URL xdcc.IRCFile
|
||||||
|
Name string
|
||||||
|
Size int64
|
||||||
|
Slot int
|
||||||
|
}
|
||||||
|
|
||||||
|
type XdccSearchProvider interface {
|
||||||
|
Search(keywords []string) ([]XdccFileInfo, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
type ProviderAggregator struct {
|
||||||
|
providerList []XdccSearchProvider
|
||||||
|
}
|
||||||
|
|
||||||
|
const MaxProviders = 100
|
||||||
|
|
||||||
|
func NewProviderAggregator(providers ...XdccSearchProvider) *ProviderAggregator {
|
||||||
|
return &ProviderAggregator{
|
||||||
|
providerList: providers,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (registry *ProviderAggregator) AddProvider(provider XdccSearchProvider) {
|
||||||
|
registry.providerList = append(registry.providerList, provider)
|
||||||
|
}
|
||||||
|
|
||||||
|
const MaxResults = 1024
|
||||||
|
|
||||||
|
func (registry *ProviderAggregator) Search(keywords []string) ([]XdccFileInfo, error) {
|
||||||
|
allResults := make(map[xdcc.IRCFile]XdccFileInfo)
|
||||||
|
|
||||||
|
mtx := sync.Mutex{}
|
||||||
|
|
||||||
|
wg := sync.WaitGroup{}
|
||||||
|
wg.Add(len(registry.providerList))
|
||||||
|
for _, p := range registry.providerList {
|
||||||
|
go func(p XdccSearchProvider) {
|
||||||
|
resList, err := p.Search(keywords)
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
mtx.Lock()
|
||||||
|
for _, res := range resList {
|
||||||
|
allResults[res.URL] = res
|
||||||
|
}
|
||||||
|
mtx.Unlock()
|
||||||
|
|
||||||
|
wg.Done()
|
||||||
|
}(p)
|
||||||
|
}
|
||||||
|
wg.Wait()
|
||||||
|
|
||||||
|
results := make([]XdccFileInfo, 0, MaxResults)
|
||||||
|
for _, res := range allResults {
|
||||||
|
results = append(results, res)
|
||||||
|
}
|
||||||
|
return results, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
const (
|
||||||
|
KiloByte = 1024
|
||||||
|
MegaByte = KiloByte * 1024
|
||||||
|
GigaByte = MegaByte * 1024
|
||||||
|
)
|
||||||
|
|
||||||
|
func parseFileSize(sizeStr string) (int64, error) {
|
||||||
|
if len(sizeStr) == 0 {
|
||||||
|
return -1, errors.New("empty string")
|
||||||
|
}
|
||||||
|
lastChar := sizeStr[len(sizeStr)-1]
|
||||||
|
sizePart := sizeStr[:len(sizeStr)-1]
|
||||||
|
|
||||||
|
size, err := strconv.ParseFloat(sizePart, 32)
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
return -1, err
|
||||||
|
}
|
||||||
|
switch lastChar {
|
||||||
|
case 'G':
|
||||||
|
return int64(size * GigaByte), nil
|
||||||
|
case 'M':
|
||||||
|
return int64(size * MegaByte), nil
|
||||||
|
case 'K':
|
||||||
|
return int64(size * KiloByte), nil
|
||||||
|
}
|
||||||
|
return -1, errors.New("unable to parse: " + sizeStr)
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,114 @@
|
||||||
|
package search
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"net/http"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
sunXdccURL = "http://sunxdcc.com/deliver.php"
|
||||||
|
sunXdccNumberOfEntries = 8
|
||||||
|
)
|
||||||
|
|
||||||
|
type SunXdccProvider struct{}
|
||||||
|
|
||||||
|
func (p *SunXdccProvider) parseResponseEntry(entry *SunXdccResponse, index int) (*XdccFileInfo, error) {
|
||||||
|
info := &XdccFileInfo{}
|
||||||
|
info.URL.Network = entry.Network[index]
|
||||||
|
info.URL.UserName = entry.Bot[index]
|
||||||
|
info.URL.Channel = entry.Channel[index]
|
||||||
|
|
||||||
|
slot, err := strconv.Atoi(entry.Packnum[index][1:])
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
sizeString := strings.TrimLeft(strings.TrimRight(entry.Fsize[index], "]"), "[")
|
||||||
|
|
||||||
|
info.Size, _ = parseFileSize(sizeString) // ignoring error
|
||||||
|
info.Name = entry.Fname[index]
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
info.Slot = slot
|
||||||
|
return info, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
type SunXdccResponse struct {
|
||||||
|
Botrec []string
|
||||||
|
Network []string
|
||||||
|
Bot []string
|
||||||
|
Channel []string
|
||||||
|
Packnum []string
|
||||||
|
Gets []string
|
||||||
|
Fsize []string
|
||||||
|
Fname []string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *SunXdccProvider) Search(keywords []string) ([]XdccFileInfo, error) {
|
||||||
|
keywordString := strings.Join(keywords, " ")
|
||||||
|
searchkey := strings.Join(strings.Fields(keywordString), "+")
|
||||||
|
// see https://sunxdcc.com/#api for API definition
|
||||||
|
httpResp, err := http.Get(sunXdccURL + "?sterm=" + searchkey)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
defer httpResp.Body.Close()
|
||||||
|
if httpResp.StatusCode != http.StatusOK {
|
||||||
|
return nil, fmt.Errorf("status code error: %d %s", httpResp.StatusCode, httpResp.Status)
|
||||||
|
}
|
||||||
|
|
||||||
|
resp, err := p.parseResponse(httpResp)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
if !p.validateResult(resp) {
|
||||||
|
return nil, fmt.Errorf("parse Error, not all fields have the same size")
|
||||||
|
}
|
||||||
|
return p.parseResults(resp)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *SunXdccProvider) parseResults(resp *SunXdccResponse) ([]XdccFileInfo, error) {
|
||||||
|
fileInfos := make([]XdccFileInfo, 0)
|
||||||
|
for i := 0; i < len(resp.Botrec); i++ {
|
||||||
|
info, err := p.parseResponseEntry(resp, i)
|
||||||
|
if err == nil {
|
||||||
|
fileInfos = append(fileInfos, *info)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return fileInfos, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (*SunXdccProvider) validateResult(entry *SunXdccResponse) bool {
|
||||||
|
sizes := [8]int{
|
||||||
|
len(entry.Botrec),
|
||||||
|
len(entry.Network),
|
||||||
|
len(entry.Bot),
|
||||||
|
len(entry.Channel),
|
||||||
|
len(entry.Packnum),
|
||||||
|
len(entry.Gets),
|
||||||
|
len(entry.Fsize),
|
||||||
|
len(entry.Fname),
|
||||||
|
}
|
||||||
|
|
||||||
|
length := sizes[0]
|
||||||
|
for _, l := range sizes {
|
||||||
|
if length != l {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
func (*SunXdccProvider) parseResponse(res *http.Response) (*SunXdccResponse, error) {
|
||||||
|
entry := &SunXdccResponse{}
|
||||||
|
decoder := json.NewDecoder(res.Body)
|
||||||
|
err := decoder.Decode(entry)
|
||||||
|
return entry, err
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,90 @@
|
||||||
|
package search
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"net/http"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"xdcc-cli/xdcc"
|
||||||
|
|
||||||
|
"github.com/PuerkitoBio/goquery"
|
||||||
|
)
|
||||||
|
|
||||||
|
type XdccEuProvider struct{}
|
||||||
|
|
||||||
|
const (
|
||||||
|
xdccEuURL = "https://www.xdcc.eu/search.php"
|
||||||
|
xdccEuNumberOfEntries = 7
|
||||||
|
)
|
||||||
|
|
||||||
|
func (p *XdccEuProvider) parseFields(fields []string) (*XdccFileInfo, error) {
|
||||||
|
if len(fields) != xdccEuNumberOfEntries {
|
||||||
|
return nil, errors.New("unexpected number of search entry fields")
|
||||||
|
}
|
||||||
|
|
||||||
|
fInfo := &XdccFileInfo{}
|
||||||
|
fInfo.URL.Network = fields[0]
|
||||||
|
fInfo.URL.Channel = fields[1]
|
||||||
|
fInfo.URL.UserName = fields[2]
|
||||||
|
slot, err := strconv.Atoi(fields[3][1:])
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
fInfo.Size, _ = parseFileSize(fields[5]) // ignoring error
|
||||||
|
|
||||||
|
fInfo.Name = fields[6]
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
fInfo.Slot = slot
|
||||||
|
return fInfo, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *XdccEuProvider) Search(keywords []string) ([]XdccFileInfo, error) {
|
||||||
|
keywordString := strings.Join(keywords, " ")
|
||||||
|
searchkey := strings.Join(strings.Fields(keywordString), "+")
|
||||||
|
res, err := http.Get(xdccEuURL + "?searchkey=" + searchkey)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
defer res.Body.Close()
|
||||||
|
if res.StatusCode != http.StatusOK {
|
||||||
|
return nil, fmt.Errorf("status code error: %d %s", res.StatusCode, res.Status)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Load the HTML document
|
||||||
|
doc, err := goquery.NewDocumentFromReader(res.Body)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
fileInfos := make([]XdccFileInfo, 0)
|
||||||
|
doc.Find("tr").Each(func(_ int, s *goquery.Selection) {
|
||||||
|
fields := make([]string, 0)
|
||||||
|
|
||||||
|
var urlStr string
|
||||||
|
s.Children().Each(func(i int, si *goquery.Selection) {
|
||||||
|
if i == 1 {
|
||||||
|
value, exists := si.Find("a").First().Attr("href")
|
||||||
|
if exists {
|
||||||
|
urlStr = value
|
||||||
|
}
|
||||||
|
}
|
||||||
|
fields = append(fields, strings.TrimSpace(si.Text()))
|
||||||
|
})
|
||||||
|
|
||||||
|
info, err := p.parseFields(fields)
|
||||||
|
if err == nil {
|
||||||
|
url, err := xdcc.ParseURL(urlStr + "/" + info.URL.UserName + "/" + strconv.Itoa(info.Slot))
|
||||||
|
if err == nil {
|
||||||
|
info.URL = *url
|
||||||
|
fileInfos = append(fileInfos, *info)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
return fileInfos, nil
|
||||||
|
}
|
||||||
|
|
@ -1,9 +1,10 @@
|
||||||
package main
|
package table
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
"sort"
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
|
"xdcc-cli/util"
|
||||||
)
|
)
|
||||||
|
|
||||||
type Row []string
|
type Row []string
|
||||||
|
|
@ -33,15 +34,8 @@ func centerString(s string, width int) string {
|
||||||
return strings.Repeat(" ", leftPadding) + s + strings.Repeat(" ", rightPadding)
|
return strings.Repeat(" ", leftPadding) + s + strings.Repeat(" ", rightPadding)
|
||||||
}
|
}
|
||||||
|
|
||||||
func cutStr(s string, maxSize int) string {
|
|
||||||
if len(s) <= maxSize {
|
|
||||||
return s
|
|
||||||
}
|
|
||||||
return s[:maxSize-3] + "..."
|
|
||||||
}
|
|
||||||
|
|
||||||
func formatStr(s string, maxSize int) string {
|
func formatStr(s string, maxSize int) string {
|
||||||
return centerString(cutStr(s, maxSize), maxSize)
|
return centerString(util.CutStr(s, maxSize), maxSize)
|
||||||
}
|
}
|
||||||
|
|
||||||
const paddingDefault = 2
|
const paddingDefault = 2
|
||||||
|
|
@ -0,0 +1,8 @@
|
||||||
|
package util
|
||||||
|
|
||||||
|
func CutStr(s string, maxSize int) string {
|
||||||
|
if len(s) <= maxSize {
|
||||||
|
return s
|
||||||
|
}
|
||||||
|
return s[:maxSize-3] + "..."
|
||||||
|
}
|
||||||
|
|
@ -1,4 +1,4 @@
|
||||||
package main
|
package xdcc
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"errors"
|
"errors"
|
||||||
|
|
@ -30,7 +30,7 @@ func parseSlot(slotStr string) (int, error) {
|
||||||
}
|
}
|
||||||
|
|
||||||
// url has the following format: irc://network/channel/bot/slot
|
// url has the following format: irc://network/channel/bot/slot
|
||||||
func parseURL(url string) (*IRCFile, error) {
|
func ParseURL(url string) (*IRCFile, error) {
|
||||||
if !strings.HasPrefix(url, "irc://") {
|
if !strings.HasPrefix(url, "irc://") {
|
||||||
return nil, errors.New("not an IRC url")
|
return nil, errors.New("not an IRC url")
|
||||||
}
|
}
|
||||||
|
|
@ -1,4 +1,4 @@
|
||||||
package main
|
package xdcc
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"bufio"
|
"bufio"
|
||||||
|
|
@ -136,7 +136,7 @@ type XdccTransfer struct {
|
||||||
events chan TransferEvent
|
events chan TransferEvent
|
||||||
}
|
}
|
||||||
|
|
||||||
func NewXdccTransfer(url IRCFile, filePath string, enableSSL bool, skipCertificateCheck bool) *XdccTransfer {
|
func NewTransfer(url IRCFile, filePath string, enableSSL bool, skipCertificateCheck bool) *XdccTransfer {
|
||||||
rand.Seed(time.Now().UTC().UnixNano())
|
rand.Seed(time.Now().UTC().UnixNano())
|
||||||
nick := IRCClientUserName + strconv.Itoa(int(rand.Uint32()))
|
nick := IRCClientUserName + strconv.Itoa(int(rand.Uint32()))
|
||||||
|
|
||||||
|
|
@ -223,8 +223,8 @@ func (transfer *XdccTransfer) PollEvents() chan TransferEvent {
|
||||||
}
|
}
|
||||||
|
|
||||||
type TransferProgessEvent struct {
|
type TransferProgessEvent struct {
|
||||||
transferBytes uint64
|
TransferBytes uint64
|
||||||
transferRate float32
|
TransferRate float32
|
||||||
}
|
}
|
||||||
|
|
||||||
const downloadBufSize = 1024
|
const downloadBufSize = 1024
|
||||||
|
|
@ -302,8 +302,8 @@ func (transfer *XdccTransfer) handleXdccSendRes(send *XdccSendRes) {
|
||||||
|
|
||||||
reader := NewSpeedMonitorReader(conn, func(dowloadedAmount int, speed float64) {
|
reader := NewSpeedMonitorReader(conn, func(dowloadedAmount int, speed float64) {
|
||||||
transfer.notifyEvent(&TransferProgessEvent{
|
transfer.notifyEvent(&TransferProgessEvent{
|
||||||
transferRate: float32(speed),
|
TransferRate: float32(speed),
|
||||||
transferBytes: uint64(dowloadedAmount),
|
TransferBytes: uint64(dowloadedAmount),
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
|
||||||
Loading…
Reference in New Issue