philip
/
xdcc-cli
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Module refactoring (#2) 

* Move main to cmd

* Add search module

* Clean-ups on search module

* Add modules xdcc, pb, table and util

* Update README
main
Stefano Scafiti 2022-10-22 14:22:20 +02:00 committed by GitHub
parent 1162bdb9fa
commit ed410faf0a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 360 additions and 340 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
README.md
View File

@ -22,20 +22,20 @@ Assuming you have the latest version of Go installed on your system, you can use
```bash
git clone https://github.com/ostafen/xdcc-cli.git
cd xdcc-cli
go build -o xdcc .
go build -o xdcc-cli cmd/main.go
```
## Usage
To initialize a file search, simply pass a list of keywords to the **search** subcommand like so:
```bash
foo@bar:~$ xdcc search keyword1 keyword2 ...
foo@bar:~$ xdcc-cli search keyword1 keyword2 ...
```
For example, to search for the latest iso of ubuntu, you could simply write:
```bash
foo@bar:~$ xdcc search ubuntu iso
foo@bar:~$ xdcc-cli search ubuntu iso
```
If the command succedeeds, a table, similar to the following, will be displayed:
@ -49,7 +49,7 @@ A part from file details, each row will contain an **url** of the form irc://net
To download one or more file, simply pass a list of url to the **get** subcommand like so:
```bash
foo@bar:~$ xdcc get url1 url2 ... [-o /path/to/an/output/directory]
foo@bar:~$ xdcc-cli get url1 url2 ... [-o /path/to/an/output/directory]
```
Alternatively, you could also specify a .txt input file, containing a list of urls (one for each line), using the **-i** switch.

65
main.go → cmd/main.go
View File

@ -9,24 +9,23 @@ import (
"strconv"
"strings"
"sync"
"xdcc-cli/pb"
"xdcc-cli/search"
table "xdcc-cli/table"
xdcc "xdcc-cli/xdcc"
)
var registry *XdccProviderRegistry = nil
var searchEngine *search.ProviderAggregator
func init() {
registry = NewProviderRegistry()
registry.AddProvider(&XdccEuProvider{})
registry.AddProvider(&SunXdccProvider{})
searchEngine = search.NewProviderAggregator(
&search.XdccEuProvider{},
&search.SunXdccProvider{},
)
}
var defaultColWidths []int = []int{100, 10, -1}
const (
KiloByte = 1024
MegaByte = KiloByte * 1024
GigaByte = MegaByte * 1024
)
func FloatToString(value float64) string {
if value-float64(int64(value)) > 0 {
return strconv.FormatFloat(value, 'f', 2, 32)
@ -39,12 +38,12 @@ func formatSize(size int64) string {
return "--"
}
if size >= GigaByte {
return FloatToString(float64(size)/float64(GigaByte)) + "GB"
} else if size >= MegaByte {
return FloatToString(float64(size)/float64(MegaByte)) + "MB"
} else if size >= KiloByte {
return FloatToString(float64(size)/float64(KiloByte)) + "KB"
if size >= search.GigaByte {
return FloatToString(float64(size)/float64(search.GigaByte)) + "GB"
} else if size >= search.MegaByte {
return FloatToString(float64(size)/float64(search.MegaByte)) + "MB"
} else if size >= search.KiloByte {
return FloatToString(float64(size)/float64(search.KiloByte)) + "KB"
}
return FloatToString(float64(size)) + "B"
}
@ -55,7 +54,7 @@ func searchCommand(args []string) {
args = parseFlags(searchCmd, args)
printer := NewTablePrinter([]string{"File Name", "Size", "URL"})
printer := table.NewTablePrinter([]string{"File Name", "Size", "URL"})
printer.SetMaxWidths(defaultColWidths)
if len(args) < 1 {
@ -63,9 +62,9 @@ func searchCommand(args []string) {
os.Exit(1)
}
res, _ := registry.Search(args)
res, _ := searchEngine.Search(args)
for _, fileInfo := range res {
printer.AddRow(Row{fileInfo.Name, formatSize(fileInfo.Size), fileInfo.URL.String()})
printer.AddRow(table.Row{fileInfo.Name, formatSize(fileInfo.Size), fileInfo.URL.String()})
}
sortColumn := 2
@ -77,22 +76,22 @@ func searchCommand(args []string) {
printer.Print()
}
func transferLoop(transfer *XdccTransfer) {
pb := NewProgressBar()
func transferLoop(transfer *xdcc.XdccTransfer) {
bar := pb.NewProgressBar()
evts := transfer.PollEvents()
quit := false
for !quit {
e := <-evts
switch evtType := e.(type) {
case *TransferStartedEvent:
pb.SetTotal(int(evtType.FileSize))
pb.SetFileName(evtType.FileName)
pb.SetState(ProgressStateDownloading)
case *TransferProgessEvent:
pb.Increment(int(evtType.transferBytes))
case *TransferCompletedEvent:
pb.SetState(ProgressStateCompleted)
case *xdcc.TransferStartedEvent:
bar.SetTotal(int(evtType.FileSize))
bar.SetFileName(evtType.FileName)
bar.SetState(pb.ProgressStateDownloading)
case *xdcc.TransferProgessEvent:
bar.Increment(int(evtType.TransferBytes))
case *xdcc.TransferCompletedEvent:
bar.SetState(pb.ProgressStateCompleted)
quit = true
}
}
@ -105,7 +104,7 @@ func suggestUnknownAuthoritySwitch(err error) {
}
}
func doTransfer(transfer *XdccTransfer) {
func doTransfer(transfer *xdcc.XdccTransfer) {
err := transfer.Start()
if err != nil {
fmt.Println(err)
@ -182,7 +181,7 @@ func getCommand(args []string) {
wg := sync.WaitGroup{}
for _, urlStr := range urlList {
if strings.HasPrefix(urlStr, "irc://") {
url, err := parseURL(urlStr)
url, err := xdcc.ParseURL(urlStr)
if err != nil {
fmt.Println(err.Error())
@ -190,8 +189,8 @@ func getCommand(args []string) {
}
wg.Add(1)
transfer := NewXdccTransfer(*url, *path, !*noSSL, *skipCertificateCheck)
go func(transfer *XdccTransfer) {
transfer := xdcc.NewTransfer(*url, *path, !*noSSL, *skipCertificateCheck)
go func(transfer *xdcc.XdccTransfer) {
doTransfer(transfer)
wg.Done()
}(transfer)

5
pb.go → pb/pb.go
View File

@ -1,7 +1,8 @@
package main
package pb
import (
"time"
"xdcc-cli/util"
"github.com/vbauerster/mpb/v7"
"github.com/vbauerster/mpb/v7/decor"
@ -38,7 +39,7 @@ const (
)
func createMpbBar(p *mpb.Progress, total int, taskName string, state ProgressState, queueBar *mpb.Bar) *mpb.Bar {
displayName := cutStr(taskName, barMaxFileNameWidth)
displayName := util.CutStr(taskName, barMaxFileNameWidth)
len := len(displayName)
if len != 0 {

284
search.go
View File

@ -1,284 +0,0 @@
package main
import (
"encoding/json"
"errors"
"fmt"
"log"
"net/http"
"strconv"
"strings"
"sync"
"github.com/PuerkitoBio/goquery"
)
type XdccFileInfo struct {
URL IRCFile
Name string
Size int64
Slot int
}
type XdccSearchProvider interface {
Search(keywords []string) ([]XdccFileInfo, error)
}
type XdccProviderRegistry struct {
providerList []XdccSearchProvider
}
const MaxProviders = 100
func NewProviderRegistry() *XdccProviderRegistry {
return &XdccProviderRegistry{
providerList: make([]XdccSearchProvider, 0, MaxProviders),
}
}
func (registry *XdccProviderRegistry) AddProvider(provider XdccSearchProvider) {
registry.providerList = append(registry.providerList, provider)
}
const MaxResults = 1024
func (registry *XdccProviderRegistry) Search(keywords []string) ([]XdccFileInfo, error) {
allResults := make(map[IRCFile]XdccFileInfo)
mtx := sync.Mutex{}
wg := sync.WaitGroup{}
wg.Add(len(registry.providerList))
for _, p := range registry.providerList {
go func(p XdccSearchProvider) {
resList, err := p.Search(keywords)
if err != nil {
return
}
mtx.Lock()
for _, res := range resList {
allResults[res.URL] = res
}
mtx.Unlock()
wg.Done()
}(p)
}
wg.Wait()
results := make([]XdccFileInfo, 0, MaxResults)
for _, res := range allResults {
results = append(results, res)
}
return results, nil
}
func parseFileSize(sizeStr string) (int64, error) {
if len(sizeStr) == 0 {
return -1, errors.New("empty string")
}
lastChar := sizeStr[len(sizeStr)-1]
sizePart := sizeStr[:len(sizeStr)-1]
size, err := strconv.ParseFloat(sizePart, 32)
if err != nil {
return -1, err
}
switch lastChar {
case 'G':
return int64(size * GigaByte), nil
case 'M':
return int64(size * MegaByte), nil
case 'K':
return int64(size * KiloByte), nil
}
return -1, errors.New("unable to parse: " + sizeStr)
}
type XdccEuProvider struct{}
const XdccEuURL = "https://www.xdcc.eu/search.php"
const xdccEuNumberOfEntries = 7
func (p *XdccEuProvider) parseFields(fields []string) (*XdccFileInfo, error) {
if len(fields) != xdccEuNumberOfEntries {
return nil, errors.New("unexpected number of search entry fields")
}
fInfo := &XdccFileInfo{}
fInfo.URL.Network = fields[0]
fInfo.URL.Channel = fields[1]
fInfo.URL.UserName = fields[2]
slot, err := strconv.Atoi(fields[3][1:])
if err != nil {
return nil, err
}
fInfo.Size, _ = parseFileSize(fields[5]) // ignoring error
fInfo.Name = fields[6]
if err != nil {
return nil, err
}
fInfo.Slot = slot
return fInfo, nil
}
func (p *XdccEuProvider) Search(keywords []string) ([]XdccFileInfo, error) {
keywordString := strings.Join(keywords, " ")
searchkey := strings.Join(strings.Fields(keywordString), "+")
res, err := http.Get(XdccEuURL + "?searchkey=" + searchkey)
if err != nil {
log.Fatal(err)
return nil, err
}
defer res.Body.Close()
if res.StatusCode != http.StatusOK {
return nil, fmt.Errorf("status code error: %d %s", res.StatusCode, res.Status)
}
// Load the HTML document
doc, err := goquery.NewDocumentFromReader(res.Body)
if err != nil {
log.Fatal(err)
return nil, err
}
fileInfos := make([]XdccFileInfo, 0)
doc.Find("tr").Each(func(_ int, s *goquery.Selection) {
fields := make([]string, 0)
var urlStr string
s.Children().Each(func(i int, si *goquery.Selection) {
if i == 1 {
value, exists := si.Find("a").First().Attr("href")
if exists {
urlStr = value
}
}
fields = append(fields, strings.TrimSpace(si.Text()))
})
info, err := p.parseFields(fields)
if err == nil {
url, err := parseURL(urlStr + "/" + info.URL.UserName + "/" + strconv.Itoa(info.Slot))
if err == nil {
info.URL = *url
fileInfos = append(fileInfos, *info)
}
}
})
return fileInfos, nil
}
const (
SunXdccURL = "http://sunxdcc.com/deliver.php"
SunXdccNumberOfEntries = 8
)
type SunXdccProvider struct{}
func (p *SunXdccProvider) parseFields(entry *SunXdccEntry, index int) (*XdccFileInfo, error) {
info := &XdccFileInfo{}
info.URL.Network = entry.Network[index]
info.URL.UserName = entry.Bot[index]
info.URL.Channel = entry.Channel[index]
slot, err := strconv.Atoi(entry.Packnum[index][1:])
if err != nil {
return nil, err
}
sizeString := strings.TrimLeft(strings.TrimRight(entry.Fsize[index], "]"), "[")
info.Size, _ = parseFileSize(sizeString) // ignoring error
info.Name = entry.Fname[index]
if err != nil {
return nil, err
}
info.Slot = slot
return info, nil
}
type SunXdccEntry struct {
Botrec []string
Network []string
Bot []string
Channel []string
Packnum []string
Gets []string
Fsize []string
Fname []string
}
func (p *SunXdccProvider) Search(keywords []string) ([]XdccFileInfo, error) {
keywordString := strings.Join(keywords, " ")
searchkey := strings.Join(strings.Fields(keywordString), "+")
// see https://sunxdcc.com/#api for API definition
res, err := http.Get(SunXdccURL + "?sterm=" + searchkey)
if err != nil {
log.Fatal(err)
return nil, err
}
defer res.Body.Close()
if res.StatusCode != http.StatusOK {
return nil, fmt.Errorf("status code error: %d %s", res.StatusCode, res.Status)
}
entry, err := p.parseResponse(res)
if err != nil {
return nil, err
}
if !p.validateResult(entry) {
return nil, fmt.Errorf("Parse Error, not all fields have the same size")
}
fileInfos := make([]XdccFileInfo, 0)
for i := 0; i < len(entry.Botrec); i++ {
info, err := p.parseFields(entry, i)
if err == nil {
fileInfos = append(fileInfos, *info)
}
}
return fileInfos, nil
}
func (*SunXdccProvider) validateResult(entry *SunXdccEntry) bool {
sizes := [8]int{
len(entry.Botrec),
len(entry.Network),
len(entry.Bot),
len(entry.Channel),
len(entry.Packnum),
len(entry.Gets),
len(entry.Fsize),
len(entry.Fname),
}
length := sizes[0]
for _, l := range sizes {
if length != l {
return false
}
}
return true
}
func (*SunXdccProvider) parseResponse(res *http.Response) (*SunXdccEntry, error) {
entry := &SunXdccEntry{}
decoder := json.NewDecoder(res.Body)
err := decoder.Decode(entry)
return entry, err
}

98
search/search.go Normal file
View File

@ -0,0 +1,98 @@
package search
import (
"errors"
"strconv"
"sync"
"xdcc-cli/xdcc"
)
type XdccFileInfo struct {
URL xdcc.IRCFile
Name string
Size int64
Slot int
}
type XdccSearchProvider interface {
Search(keywords []string) ([]XdccFileInfo, error)
}
type ProviderAggregator struct {
providerList []XdccSearchProvider
}
const MaxProviders = 100
func NewProviderAggregator(providers ...XdccSearchProvider) *ProviderAggregator {
return &ProviderAggregator{
providerList: providers,
}
}
func (registry *ProviderAggregator) AddProvider(provider XdccSearchProvider) {
registry.providerList = append(registry.providerList, provider)
}
const MaxResults = 1024
func (registry *ProviderAggregator) Search(keywords []string) ([]XdccFileInfo, error) {
allResults := make(map[xdcc.IRCFile]XdccFileInfo)
mtx := sync.Mutex{}
wg := sync.WaitGroup{}
wg.Add(len(registry.providerList))
for _, p := range registry.providerList {
go func(p XdccSearchProvider) {
resList, err := p.Search(keywords)
if err != nil {
return
}
mtx.Lock()
for _, res := range resList {
allResults[res.URL] = res
}
mtx.Unlock()
wg.Done()
}(p)
}
wg.Wait()
results := make([]XdccFileInfo, 0, MaxResults)
for _, res := range allResults {
results = append(results, res)
}
return results, nil
}
const (
KiloByte = 1024
MegaByte = KiloByte * 1024
GigaByte = MegaByte * 1024
)
func parseFileSize(sizeStr string) (int64, error) {
if len(sizeStr) == 0 {
return -1, errors.New("empty string")
}
lastChar := sizeStr[len(sizeStr)-1]
sizePart := sizeStr[:len(sizeStr)-1]
size, err := strconv.ParseFloat(sizePart, 32)
if err != nil {
return -1, err
}
switch lastChar {
case 'G':
return int64(size * GigaByte), nil
case 'M':
return int64(size * MegaByte), nil
case 'K':
return int64(size * KiloByte), nil
}
return -1, errors.New("unable to parse: " + sizeStr)
}

114
search/sun_xdcc.go Normal file
View File

@ -0,0 +1,114 @@
package search
import (
"encoding/json"
"fmt"
"net/http"
"strconv"
"strings"
)
const (
sunXdccURL = "http://sunxdcc.com/deliver.php"
sunXdccNumberOfEntries = 8
)
type SunXdccProvider struct{}
func (p *SunXdccProvider) parseResponseEntry(entry *SunXdccResponse, index int) (*XdccFileInfo, error) {
info := &XdccFileInfo{}
info.URL.Network = entry.Network[index]
info.URL.UserName = entry.Bot[index]
info.URL.Channel = entry.Channel[index]
slot, err := strconv.Atoi(entry.Packnum[index][1:])
if err != nil {
return nil, err
}
sizeString := strings.TrimLeft(strings.TrimRight(entry.Fsize[index], "]"), "[")
info.Size, _ = parseFileSize(sizeString) // ignoring error
info.Name = entry.Fname[index]
if err != nil {
return nil, err
}
info.Slot = slot
return info, nil
}
type SunXdccResponse struct {
Botrec []string
Network []string
Bot []string
Channel []string
Packnum []string
Gets []string
Fsize []string
Fname []string
}
func (p *SunXdccProvider) Search(keywords []string) ([]XdccFileInfo, error) {
keywordString := strings.Join(keywords, " ")
searchkey := strings.Join(strings.Fields(keywordString), "+")
// see https://sunxdcc.com/#api for API definition
httpResp, err := http.Get(sunXdccURL + "?sterm=" + searchkey)
if err != nil {
return nil, err
}
defer httpResp.Body.Close()
if httpResp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("status code error: %d %s", httpResp.StatusCode, httpResp.Status)
}
resp, err := p.parseResponse(httpResp)
if err != nil {
return nil, err
}
if !p.validateResult(resp) {
return nil, fmt.Errorf("parse Error, not all fields have the same size")
}
return p.parseResults(resp)
}
func (p *SunXdccProvider) parseResults(resp *SunXdccResponse) ([]XdccFileInfo, error) {
fileInfos := make([]XdccFileInfo, 0)
for i := 0; i < len(resp.Botrec); i++ {
info, err := p.parseResponseEntry(resp, i)
if err == nil {
fileInfos = append(fileInfos, *info)
}
}
return fileInfos, nil
}
func (*SunXdccProvider) validateResult(entry *SunXdccResponse) bool {
sizes := [8]int{
len(entry.Botrec),
len(entry.Network),
len(entry.Bot),
len(entry.Channel),
len(entry.Packnum),
len(entry.Gets),
len(entry.Fsize),
len(entry.Fname),
}
length := sizes[0]
for _, l := range sizes {
if length != l {
return false
}
}
return true
}
func (*SunXdccProvider) parseResponse(res *http.Response) (*SunXdccResponse, error) {
entry := &SunXdccResponse{}
decoder := json.NewDecoder(res.Body)
err := decoder.Decode(entry)
return entry, err
}

90
search/xdcc_eu.go Normal file
View File

@ -0,0 +1,90 @@
package search
import (
"errors"
"fmt"
"net/http"
"strconv"
"strings"
"xdcc-cli/xdcc"
"github.com/PuerkitoBio/goquery"
)
type XdccEuProvider struct{}
const (
xdccEuURL = "https://www.xdcc.eu/search.php"
xdccEuNumberOfEntries = 7
)
func (p *XdccEuProvider) parseFields(fields []string) (*XdccFileInfo, error) {
if len(fields) != xdccEuNumberOfEntries {
return nil, errors.New("unexpected number of search entry fields")
}
fInfo := &XdccFileInfo{}
fInfo.URL.Network = fields[0]
fInfo.URL.Channel = fields[1]
fInfo.URL.UserName = fields[2]
slot, err := strconv.Atoi(fields[3][1:])
if err != nil {
return nil, err
}
fInfo.Size, _ = parseFileSize(fields[5]) // ignoring error
fInfo.Name = fields[6]
if err != nil {
return nil, err
}
fInfo.Slot = slot
return fInfo, nil
}
func (p *XdccEuProvider) Search(keywords []string) ([]XdccFileInfo, error) {
keywordString := strings.Join(keywords, " ")
searchkey := strings.Join(strings.Fields(keywordString), "+")
res, err := http.Get(xdccEuURL + "?searchkey=" + searchkey)
if err != nil {
return nil, err
}
defer res.Body.Close()
if res.StatusCode != http.StatusOK {
return nil, fmt.Errorf("status code error: %d %s", res.StatusCode, res.Status)
}
// Load the HTML document
doc, err := goquery.NewDocumentFromReader(res.Body)
if err != nil {
return nil, err
}
fileInfos := make([]XdccFileInfo, 0)
doc.Find("tr").Each(func(_ int, s *goquery.Selection) {
fields := make([]string, 0)
var urlStr string
s.Children().Each(func(i int, si *goquery.Selection) {
if i == 1 {
value, exists := si.Find("a").First().Attr("href")
if exists {
urlStr = value
}
}
fields = append(fields, strings.TrimSpace(si.Text()))
})
info, err := p.parseFields(fields)
if err == nil {
url, err := xdcc.ParseURL(urlStr + "/" + info.URL.UserName + "/" + strconv.Itoa(info.Slot))
if err == nil {
info.URL = *url
fileInfos = append(fileInfos, *info)
}
}
})
return fileInfos, nil
}

12
table.go → table/table.go
View File

@ -1,9 +1,10 @@
package main
package table
import (
"fmt"
"sort"
"strings"
"xdcc-cli/util"
)
type Row []string
@ -33,15 +34,8 @@ func centerString(s string, width int) string {
return strings.Repeat(" ", leftPadding) + s + strings.Repeat(" ", rightPadding)
}
func cutStr(s string, maxSize int) string {
if len(s) <= maxSize {
return s
}
return s[:maxSize-3] + "..."
}
func formatStr(s string, maxSize int) string {
return centerString(cutStr(s, maxSize), maxSize)
return centerString(util.CutStr(s, maxSize), maxSize)
}
const paddingDefault = 2

8
util/format.go Normal file
View File

@ -0,0 +1,8 @@
package util
func CutStr(s string, maxSize int) string {
if len(s) <= maxSize {
return s
}
return s[:maxSize-3] + "..."
}

4
url.go → xdcc/url.go
View File

@ -1,4 +1,4 @@
package main
package xdcc
import (
"errors"
@ -30,7 +30,7 @@ func parseSlot(slotStr string) (int, error) {
}
// url has the following format: irc://network/channel/bot/slot
func parseURL(url string) (*IRCFile, error) {
func ParseURL(url string) (*IRCFile, error) {
if !strings.HasPrefix(url, "irc://") {
return nil, errors.New("not an IRC url")
}

12
xdcc.go → xdcc/xdcc.go
View File

@ -1,4 +1,4 @@
package main
package xdcc
import (
"bufio"
@ -136,7 +136,7 @@ type XdccTransfer struct {
events chan TransferEvent
}
func NewXdccTransfer(url IRCFile, filePath string, enableSSL bool, skipCertificateCheck bool) *XdccTransfer {
func NewTransfer(url IRCFile, filePath string, enableSSL bool, skipCertificateCheck bool) *XdccTransfer {
rand.Seed(time.Now().UTC().UnixNano())
nick := IRCClientUserName + strconv.Itoa(int(rand.Uint32()))
@ -223,8 +223,8 @@ func (transfer *XdccTransfer) PollEvents() chan TransferEvent {
}
type TransferProgessEvent struct {
transferBytes uint64
transferRate float32
TransferBytes uint64
TransferRate float32
}
const downloadBufSize = 1024
@ -302,8 +302,8 @@ func (transfer *XdccTransfer) handleXdccSendRes(send *XdccSendRes) {
reader := NewSpeedMonitorReader(conn, func(dowloadedAmount int, speed float64) {
transfer.notifyEvent(&TransferProgessEvent{
transferRate: float32(speed),
transferBytes: uint64(dowloadedAmount),
TransferRate: float32(speed),
TransferBytes: uint64(dowloadedAmount),
})
})