chore: Add ScrapeURL flag for CLI to scrape website URL to markdown using Jina AI

cli/cli.go

@@ -3,6 +3,7 @@ package cli
 import (
 	"fmt"
 	"os"
+	"os/exec"
 	"path/filepath"
 	"strconv"
 
@@ -94,6 +95,21 @@ func Cli() (message string, err error) {
 		return
 	}
 
+	// Check for ScrapeURL flag first
+	if currentFlags.ScrapeURL != "" {
+		fmt.Println("ScrapeURL flag is set") // Debug print
+		url := currentFlags.ScrapeURL
+		curlCommand := fmt.Sprintf("curl https://r.jina.ai/%s", url)
+		fmt.Println("Executing command:", curlCommand) // Debug print
+		if err := exec.Command("sh", "-c", curlCommand).Run(); err != nil {
+			return "", fmt.Errorf("failed to run curl command: %w", err)
+		}
+		fmt.Println("Curl command executed successfully") // Debug print
+		os.Exit(0)
+	} else {
+		fmt.Println("ScrapeURL flag is not set") // Debug print
+	}
+
 	// if the interactive flag is set, run the interactive function
 	// if currentFlags.Interactive {
 	// 	interactive.Interactive()

cli/flags.go

@@ -34,36 +34,38 @@ type Flags struct {
 	Output                  string  `short:"o" long:"output" description:"Output to file" default:""`
 	LatestPatterns          string  `short:"n" long:"latest" description:"Number of latest patterns to list" default:"0"`
 	ChangeDefaultModel      bool    `short:"d" long:"changeDefaultModel" description:"Change default pattern"`
+    ScrapeURL               string  `short:"u" long:"scrape_url" description:"Scrape website URL to markdown using Jina AI"`
+
 }
 
 // Init Initialize flags. returns a Flags struct and an error
 func Init() (ret *Flags, err error) {
-	var message string
+    var message string
 
-	ret = &Flags{}
-	parser := flags.NewParser(ret, flags.Default)
-	var args []string
-	if args, err = parser.Parse(); err != nil {
-		return
-	}
+    ret = &Flags{}
+    parser := flags.NewParser(ret, flags.Default)
+    var args []string
+    if args, err = parser.Parse(); err != nil {
+        return
+    }
 
-	info, _ := os.Stdin.Stat()
-	hasStdin := (info.Mode() & os.ModeCharDevice) == 0
+    info, _ := os.Stdin.Stat()
+    hasStdin := (info.Mode() & os.ModeCharDevice) == 0
 
-	// takes input from stdin if it exists, otherwise takes input from args (the last argument)
-	if hasStdin {
-		if message, err = readStdin(); err != nil {
-			err = errors.New("error: could not read from stdin")
-			return
-		}
-	} else if len(args) > 0 {
-		message = args[len(args)-1]
-	} else {
-		message = ""
-	}
-	ret.Message = message
+    // takes input from stdin if it exists, otherwise takes input from args (the last argument)
+    if hasStdin {
+        if message, err = readStdin(); err != nil {
+            err = errors.New("error: could not read from stdin")
+            return
+        }
+    } else if len(args) > 0 {
+        message = args[len(args)-1]
+    } else {
+        message = ""
+    }
+    ret.Message = message
 
-	return
+    return
 }
 
 // readStdin reads from stdin and returns the input as a string or an error