From b92643eebf36cbdb434f551ba2d2bb46c6b5a94a Mon Sep 17 00:00:00 2001 From: arsam Date: Fri, 18 Sep 2026 23:18:51 +0330 Subject: [PATCH] Import source: cmd/main Import the existing local project snapshot. This commit organizes existing files; it does not represent newly developed functionality. --- cmd/main.go | 138 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 138 insertions(+) create mode 100644 cmd/main.go diff --git a/cmd/main.go b/cmd/main.go new file mode 100644 index 0000000..ee6ab52 --- /dev/null +++ b/cmd/main.go @@ -0,0 +1,138 @@ +package main + +import ( + "fmt" + "log" + "net/http" + "net/url" + "os" + "strings" + "time" + + "reactor-crw" + "reactor-crw/handler/fs" + "reactor-crw/parser" + + "github.com/vbauerster/mpb/v7" + "github.com/vbauerster/mpb/v7/decor" + + "github.com/spf13/cobra" +) + +var ( + search string + path string + savePath string + cookie string + maxWorkers int + singlePage bool + + crawlerCmd = &cobra.Command{ + Use: "reactor-crw", + Short: "Grab your favorite content from joyreactor.cc", + Long: "Allows to quickly download all content by its direct url or entire" + + " tag or fandom from joyreactor.cc.\nExample: reactor-crw -d \".\" " + + "-p \"http://joyreactor.cc/tag/someTag/all\" -w 2 -c \"cookie-string\"", + Run: run, + } +) + +func init() { + hd, _ := os.UserHomeDir() + + crawlerCmd.Flags().StringVarP(&search, "search", "s", "image,gif", "A comma separated list of content types that should be downloaded.\nPossible values: image,gif,webm,mp4. Example: -s \"image,webm\"") + crawlerCmd.Flags().StringVarP(&path, "path", "p", "", "Provide a full page URL") + crawlerCmd.Flags().StringVarP(&savePath, "destination", "d", hd, "Save path for content. Default value is a user's home folder \n(example C:\\Users\\username for Windows)") + crawlerCmd.Flags().StringVarP(&cookie, "cookie", "c", "", "User's cookie. Some content may be unavailable without it") + crawlerCmd.Flags().IntVarP(&maxWorkers, "workers", "w", 1, "Amount of workers") + crawlerCmd.Flags().BoolVarP(&singlePage, "single-page", "o", false, "Crawl only one page") + + _ = crawlerCmd.MarkFlagRequired("path") +} + +func run(_ *cobra.Command, _ []string) { + start := time.Now() + t := reactor_crw.NewHttpTransport(&http.Client{}, reactor_crw.Headers{"Cookie": cookie}) + + pathUrl, err := url.Parse(path) + if err != nil { + log.Fatalf("invalid path provided: %s", path) + } + + pr, err := fs.NewPathResolver(savePath) + if err != nil { + log.Fatalf("cannot process provided destination: %s", savePath) + } + + ch, _ := fs.NewFileSaver(pr, t, strings.Replace(pathUrl.Path, "/", "_", -1)) + c := reactor_crw.NewClient( + &reactor_crw.HtmlCrawler{ + Transport: t, + Parser: &parser.Html{}, + MultiPage: !singlePage, + }, + maxWorkers, + ch, + ) + + go func() { + err = c.Run(path, search) + if err != nil { + log.Fatal(err) + } + }() + + progress(c.TotalSources, c.Progress, c.Errors) + + fmt.Printf("\n>>> Done in %s\n", time.Since(start).String()) +} + +func progress(total, task <-chan int, err <-chan error) { + fmt.Print("\n>>> Trying to count the amount of links. Please wait...\n\n") + + t := <-total + if t == 0 { + fmt.Print(">>> No links were found. Stopping...\n") + return + } + fmt.Printf(">>> %d links were found. Start downloading\n\n", t) + + prg := mpb.New(mpb.WithWidth(64)) + + name := ">>> Progress:" + bar := prg.AddBar(int64(t), + mpb.PrependDecorators( + decor.Name(name, decor.WC{W: len(name) + 1, C: decor.DidentRight}), + decor.CountersNoUnit("%d/%d", decor.WCSyncWidth), + ), + mpb.AppendDecorators(decor.Percentage(decor.WC{W: 5})), + ) + + errBuf := strings.Builder{} + + go func() { + for { + select { + case <-task: + bar.Increment() + case e := <-err: + if e != nil { + errBuf.WriteString(e.Error() + "\n") + } + } + } + }() + + prg.Wait() + + if errBuf.Len() != 0 { + fmt.Printf(">>> Errors during crawler work:\n%s", errBuf.String()) + } +} + +func main() { + err := crawlerCmd.Execute() + if err != nil { + log.Fatal(err) + } +}