chore: Add ScrapeURL flag for CLI to scrape website URL to markdown using Jina AI

This commit is contained in:
Noam Siegel 2024-08-21 10:44:57 -07:00
parent a81e5be74b
commit c7449c68b7
2 changed files with 40 additions and 22 deletions

View File

@ -3,6 +3,7 @@ package cli
import ( import (
"fmt" "fmt"
"os" "os"
"os/exec"
"path/filepath" "path/filepath"
"strconv" "strconv"
@ -94,6 +95,21 @@ func Cli() (message string, err error) {
return return
} }
// Check for ScrapeURL flag first
if currentFlags.ScrapeURL != "" {
fmt.Println("ScrapeURL flag is set") // Debug print
url := currentFlags.ScrapeURL
curlCommand := fmt.Sprintf("curl https://r.jina.ai/%s", url)
fmt.Println("Executing command:", curlCommand) // Debug print
if err := exec.Command("sh", "-c", curlCommand).Run(); err != nil {
return "", fmt.Errorf("failed to run curl command: %w", err)
}
fmt.Println("Curl command executed successfully") // Debug print
os.Exit(0)
} else {
fmt.Println("ScrapeURL flag is not set") // Debug print
}
// if the interactive flag is set, run the interactive function // if the interactive flag is set, run the interactive function
// if currentFlags.Interactive { // if currentFlags.Interactive {
// interactive.Interactive() // interactive.Interactive()

View File

@ -34,36 +34,38 @@ type Flags struct {
Output string `short:"o" long:"output" description:"Output to file" default:""` Output string `short:"o" long:"output" description:"Output to file" default:""`
LatestPatterns string `short:"n" long:"latest" description:"Number of latest patterns to list" default:"0"` LatestPatterns string `short:"n" long:"latest" description:"Number of latest patterns to list" default:"0"`
ChangeDefaultModel bool `short:"d" long:"changeDefaultModel" description:"Change default pattern"` ChangeDefaultModel bool `short:"d" long:"changeDefaultModel" description:"Change default pattern"`
ScrapeURL string `short:"u" long:"scrape_url" description:"Scrape website URL to markdown using Jina AI"`
} }
// Init Initialize flags. returns a Flags struct and an error // Init Initialize flags. returns a Flags struct and an error
func Init() (ret *Flags, err error) { func Init() (ret *Flags, err error) {
var message string var message string
ret = &Flags{} ret = &Flags{}
parser := flags.NewParser(ret, flags.Default) parser := flags.NewParser(ret, flags.Default)
var args []string var args []string
if args, err = parser.Parse(); err != nil { if args, err = parser.Parse(); err != nil {
return return
} }
info, _ := os.Stdin.Stat() info, _ := os.Stdin.Stat()
hasStdin := (info.Mode() & os.ModeCharDevice) == 0 hasStdin := (info.Mode() & os.ModeCharDevice) == 0
// takes input from stdin if it exists, otherwise takes input from args (the last argument) // takes input from stdin if it exists, otherwise takes input from args (the last argument)
if hasStdin { if hasStdin {
if message, err = readStdin(); err != nil { if message, err = readStdin(); err != nil {
err = errors.New("error: could not read from stdin") err = errors.New("error: could not read from stdin")
return return
} }
} else if len(args) > 0 { } else if len(args) > 0 {
message = args[len(args)-1] message = args[len(args)-1]
} else { } else {
message = "" message = ""
} }
ret.Message = message ret.Message = message
return return
} }
// readStdin reads from stdin and returns the input as a string or an error // readStdin reads from stdin and returns the input as a string or an error