diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..74280d4 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,25 @@ +name: Go Build and Release + +on: + push: + branches: ["main"] + pull_request: + branches: ["main"] + +jobs: + test: + name: Run tests + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Go + uses: actions/setup-go@v4 + with: + go-version-file: ./go.mod + + - name: Run tests + run: go test -v ./... diff --git a/.github/workflows/go.yml b/.github/workflows/release.yml similarity index 85% rename from .github/workflows/go.yml rename to .github/workflows/release.yml index 6284d68..3726e58 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/release.yml @@ -5,10 +5,25 @@ on: branches: ["main"] tags: - "v*" - pull_request: - branches: ["main"] jobs: + test: + name: Run tests + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Go + uses: actions/setup-go@v4 + with: + go-version-file: ./go.mod + + - name: Run tests + run: go test -v ./... + build: name: Build binaries for Windows, macOS, and Linux runs-on: ${{ matrix.os }} diff --git a/README.md b/README.md index f59509b..eddcf16 100644 --- a/README.md +++ b/README.md @@ -47,10 +47,16 @@
> [!NOTE] -August 16, 2024 — We have migrated to Go! The biggest thing to know is that **the previous installation instructions in the various Fabric videos out there will no longer work** because they were for the legacy (Python) version. Check the new [install instructions](Installation) below. +August 20, 2024 — We have migrated to Go, and the transition has been pretty smooth! The biggest thing to know is that **the previous installation instructions in the various Fabric videos out there will no longer work** because they were for the legacy (Python) version. Check the new [install instructions](#Installation) below. -> [!NOTE] -August 16, 2024 — We have cleaned up the Pull Requests and Issues in the following ways as part of the Go release: 1) We incorporated all Pattern submissions in the new version. 2) We closed all Issues related to Python/Code because we we moved to Go. If your issue still persists, just resubmit and we'll get on it. 3) We did the same with Question issues because most of them were related to Python. 4) We left the Enhancement issues because those tend to not relate as much to Python vs. Go, and we'll be working through those. +## Intro videos + +Keep in mind that many of these were recorded when Fabric was Python-based, so remember to use the current [install instructions](#Installation) below. + +* [Network Chuck](https://www.youtube.com/watch?v=UbDyjIIGaxQ) +* [David Bombal](https://www.youtube.com/watch?v=vF-MQmVxnCs) +* [My Own Intro to the Tool](https://www.youtube.com/watch?v=wPEyyigh10g) +* [More Fabric YouTube Videos](https://www.youtube.com/results?search_query=fabric+ai) ## What and why @@ -100,8 +106,10 @@ To install Fabric, [make sure Go is installed](https://go.dev/doc/install), and ```bash # Install Fabric directly from the repo go install github.com/danielmiessler/fabric@latest + # Run the setup to set up your directories and keys fabric --setup +``` ### Environment Variables @@ -130,16 +138,7 @@ go install github.com/danielmiessler/fabric@latest fabric --setup ``` -### Environment Variables - -If everything works you are good to go, but you may need to set some environment variables in your `~/.bashrc` or `~/.zshrc` file. Here is an example of what you can add: - -```bash -# Golang environment variables -export GOROOT=/usr/local/go -export GOPATH=$HOME/go -export PATH=$GOPATH/bin:$GOROOT/bin:$HOME/.local/bin:$PATH: -``` +Then [set your environmental variables](#environmental-variables) as shown above. ### Upgrading @@ -151,8 +150,9 @@ go install github.com/danielmiessler/fabric@latest ## Usage Once you have it all set up, here's how to use it. -1. Check out the options - `fabric -h` +```bash +fabric -h +``` ```bash usage: fabric -h @@ -183,34 +183,6 @@ Application Options: Help Options: -h, --help Show this help message -Usage: - fabric [OPTIONS] - -Application Options: - -p, --pattern= Choose a pattern - -C, --context= Choose a context - --session= Choose a session - -S, --setup Run setup - -t, --temperature= Set temperature (default: 0.7) - -T, --topp= Set top P (default: 0.9) - -s, --stream Stream - -P, --presencepenalty= Set presence penalty (default: 0.0) - -F, --frequencypenalty= Set frequency penalty (default: 0.0) - -l, --listpatterns List all patterns - -L, --listmodels List all available models - -x, --listcontexts List all contexts - -X, --listsessions List all sessions - -U, --updatepatterns Update patterns - -A, --addcontext Add a context - -c, --copy Copy to clipboard - -m, --model= Choose model - -u, --url= Choose ollama url (default: http://127.0.0.1:11434) - -o, --output= Output to file - -n, --latest= Number of latest patterns to list (default: 0) - -Help Options: - -h, --help Show this help message -``` ``` ## Our approach to prompting @@ -233,6 +205,8 @@ https://github.com/danielmiessler/fabric/blob/main/patterns/extract_wisdom/syste ## Examples +Now let's look at some things you can do with Fabric. + 1. Run the `summarize` Pattern based on input from `stdin`. In this case, the body of an article. ```bash @@ -251,7 +225,7 @@ pbpaste | fabric --stream --pattern analyze_claims yt --transcript https://youtube.com/watch?v=uXs-zPc63kM | fabric --stream --pattern extract_wisdom ``` -4. Create patterns- you must create a .md file with the pattern and save it to ~/.config/fabric/pattterns/[yourpatternname]. +4. Create patterns- you must create a .md file with the pattern and save it to ~/.config/fabric/patterns/[yourpatternname]. ## Just use the Patterns diff --git a/cli/cli.go b/cli/cli.go index 6028e65..f8178de 100644 --- a/cli/cli.go +++ b/cli/cli.go @@ -5,6 +5,7 @@ import ( "os" "path/filepath" "strconv" + "strings" "github.com/danielmiessler/fabric/core" "github.com/danielmiessler/fabric/db" @@ -14,7 +15,7 @@ import ( func Cli() (message string, err error) { var currentFlags *Flags if currentFlags, err = Init(); err != nil { - // we need to reset error, because we want to show double help messages + // we need to reset error, because we don't want to show double help messages err = nil return } @@ -24,23 +25,23 @@ func Cli() (message string, err error) { return } - db := db.NewDb(filepath.Join(homedir, ".config/fabric")) + fabricDb := db.NewDb(filepath.Join(homedir, ".config/fabric")) // if the setup flag is set, run the setup function if currentFlags.Setup { - _ = db.Configure() - _, err = Setup(db, currentFlags.SetupSkipUpdatePatterns) + _ = fabricDb.Configure() + _, err = Setup(fabricDb, currentFlags.SetupSkipUpdatePatterns) return } var fabric *core.Fabric - if err = db.Configure(); err != nil { + if err = fabricDb.Configure(); err != nil { fmt.Println("init is failed, run start the setup procedure", err) - if fabric, err = Setup(db, currentFlags.SetupSkipUpdatePatterns); err != nil { + if fabric, err = Setup(fabricDb, currentFlags.SetupSkipUpdatePatterns); err != nil { return } } else { - if fabric, err = core.NewFabric(db); err != nil { + if fabric, err = core.NewFabric(fabricDb); err != nil { fmt.Println("fabric can't initialize, please run the --setup procedure", err) return } @@ -64,7 +65,7 @@ func Cli() (message string, err error) { return } - if err = db.Patterns.PrintLatestPatterns(parsedToInt); err != nil { + if err = fabricDb.Patterns.PrintLatestPatterns(parsedToInt); err != nil { return } return @@ -72,7 +73,7 @@ func Cli() (message string, err error) { // if the list patterns flag is set, run the list all patterns function if currentFlags.ListPatterns { - err = db.Patterns.ListNames() + err = fabricDb.Patterns.ListNames() return } @@ -84,13 +85,13 @@ func Cli() (message string, err error) { // if the list all contexts flag is set, run the list all contexts function if currentFlags.ListAllContexts { - err = db.Contexts.ListNames() + err = fabricDb.Contexts.ListNames() return } // if the list all sessions flag is set, run the list all sessions function if currentFlags.ListAllSessions { - err = db.Sessions.ListNames() + err = fabricDb.Sessions.ListNames() return } @@ -101,6 +102,46 @@ func Cli() (message string, err error) { // if none of the above currentFlags are set, run the initiate chat function + if currentFlags.YouTube != "" { + if fabric.YouTube.IsConfigured() == false { + err = fmt.Errorf("YouTube is not configured, please run the setup procedure") + return + } + + var videoId string + if videoId, err = fabric.YouTube.GetVideoId(currentFlags.YouTube); err != nil { + return + } + + if currentFlags.YouTubeTranscript { + var transcript string + if transcript, err = fabric.YouTube.GrabTranscript(videoId); err != nil { + return + } + + if currentFlags.Message != "" { + currentFlags.Message = currentFlags.Message + "\n" + transcript + } else { + currentFlags.Message = transcript + } + } + + if currentFlags.YouTubeComments { + var comments []string + if comments, err = fabric.YouTube.GrabComments(videoId); err != nil { + return + } + + commentsString := strings.Join(comments, "\n") + + if currentFlags.Message != "" { + currentFlags.Message = currentFlags.Message + "\n" + commentsString + } else { + currentFlags.Message = commentsString + } + } + } + var chatter *core.Chatter if chatter, err = fabric.GetChatter(currentFlags.Model, currentFlags.Stream); err != nil { return @@ -129,17 +170,17 @@ func Cli() (message string, err error) { } func Setup(db *db.Db, skipUpdatePatterns bool) (ret *core.Fabric, err error) { - ret = core.NewFabricForSetup(db) + instance := core.NewFabricForSetup(db) - if err = ret.Setup(); err != nil { + if err = instance.Setup(); err != nil { return } if !skipUpdatePatterns { - if err = ret.PopulateDB(); err != nil { + if err = instance.PopulateDB(); err != nil { return } } - + ret = instance return } diff --git a/cli/cli_test.go b/cli/cli_test.go new file mode 100644 index 0000000..95b8701 --- /dev/null +++ b/cli/cli_test.go @@ -0,0 +1,23 @@ +package cli + +import ( + "os" + "testing" + + "github.com/danielmiessler/fabric/db" + "github.com/stretchr/testify/assert" +) + +func TestCli(t *testing.T) { + message, err := Cli() + assert.NoError(t, err) + assert.Empty(t, message) +} + +func TestSetup(t *testing.T) { + mockDB := db.NewDb(os.TempDir()) + + fabric, err := Setup(mockDB, false) + assert.Error(t, err) + assert.Nil(t, fabric) +} diff --git a/cli/flags.go b/cli/flags.go index c4fde8d..ee8bdd4 100644 --- a/cli/flags.go +++ b/cli/flags.go @@ -34,6 +34,9 @@ type Flags struct { Output string `short:"o" long:"output" description:"Output to file" default:""` LatestPatterns string `short:"n" long:"latest" description:"Number of latest patterns to list" default:"0"` ChangeDefaultModel bool `short:"d" long:"changeDefaultModel" description:"Change default pattern"` + YouTube string `short:"y" long:"youtube" description:"YouTube video url to grab transcript, comments from it and send to chat"` + YouTubeTranscript bool `long:"transcript" description:"Grab transcript from YouTube video and send to chat"` + YouTubeComments bool `long:"comments" description:"Grab comments from YouTube video and send to chat"` } // Init Initialize flags. returns a Flags struct and an error diff --git a/cli/flags_test.go b/cli/flags_test.go new file mode 100644 index 0000000..992d70d --- /dev/null +++ b/cli/flags_test.go @@ -0,0 +1,85 @@ +package cli + +import ( + "bytes" + "io" + "io/ioutil" + "os" + "strings" + "testing" + + "github.com/danielmiessler/fabric/common" + "github.com/stretchr/testify/assert" +) + +func TestInit(t *testing.T) { + args := []string{"--copy"} + expectedFlags := &Flags{Copy: true} + oldArgs := os.Args + defer func() { os.Args = oldArgs }() + os.Args = append([]string{"cmd"}, args...) + + flags, err := Init() + assert.NoError(t, err) + assert.Equal(t, expectedFlags.Copy, flags.Copy) +} + +func TestReadStdin(t *testing.T) { + input := "test input" + stdin := ioutil.NopCloser(strings.NewReader(input)) + // No need to cast stdin to *os.File, pass it as io.ReadCloser directly + content, err := ReadStdin(stdin) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if content != input { + t.Fatalf("expected %q, got %q", input, content) + } +} + +// ReadStdin function assuming it's part of `cli` package +func ReadStdin(reader io.ReadCloser) (string, error) { + defer reader.Close() + buf := new(bytes.Buffer) + _, err := buf.ReadFrom(reader) + if err != nil { + return "", err + } + return buf.String(), nil +} + +func TestBuildChatOptions(t *testing.T) { + flags := &Flags{ + Temperature: 0.8, + TopP: 0.9, + PresencePenalty: 0.1, + FrequencyPenalty: 0.2, + } + + expectedOptions := &common.ChatOptions{ + Temperature: 0.8, + TopP: 0.9, + PresencePenalty: 0.1, + FrequencyPenalty: 0.2, + } + options := flags.BuildChatOptions() + assert.Equal(t, expectedOptions, options) +} + +func TestBuildChatRequest(t *testing.T) { + flags := &Flags{ + Context: "test-context", + Session: "test-session", + Pattern: "test-pattern", + Message: "test-message", + } + + expectedRequest := &common.ChatRequest{ + ContextName: "test-context", + SessionName: "test-session", + PatternName: "test-pattern", + Message: "test-message", + } + request := flags.BuildChatRequest() + assert.Equal(t, expectedRequest, request) +} diff --git a/common/configurable.go b/common/configurable.go index 0ec61d0..172d7c5 100644 --- a/common/configurable.go +++ b/common/configurable.go @@ -23,10 +23,6 @@ func (o *Configurable) GetName() string { return o.Label } -func (o *Configurable) GetSettings() Settings { - return o.Settings -} - func (o *Configurable) AddSetting(name string, required bool) (ret *Setting) { ret = NewSetting(fmt.Sprintf("%v%v", o.EnvNamePrefix, BuildEnvVariable(name)), required) o.Settings = append(o.Settings, ret) @@ -67,6 +63,17 @@ func (o *Configurable) Setup() (err error) { return } +func (o *Configurable) SetupOrSkip() (err error) { + if err = o.Setup(); err != nil { + fmt.Printf("[%v] skipped\n", o.GetName()) + } + return +} + +func (o *Configurable) SetupFillEnvFileContent(fileEnvFileContent *bytes.Buffer) { + o.Settings.FillEnvFileContent(fileEnvFileContent) +} + func NewSetting(envVariable string, required bool) *Setting { return &Setting{ EnvVariable: envVariable, diff --git a/common/configurable_test.go b/common/configurable_test.go new file mode 100644 index 0000000..3ec2560 --- /dev/null +++ b/common/configurable_test.go @@ -0,0 +1,176 @@ +package common + +import ( + "bytes" + "os" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestConfigurable_AddSetting(t *testing.T) { + conf := &Configurable{ + Settings: Settings{}, + Label: "TestConfigurable", + EnvNamePrefix: "TEST_", + } + + setting := conf.AddSetting("test_setting", true) + assert.Equal(t, "TEST_TEST_SETTING", setting.EnvVariable) + assert.True(t, setting.Required) + assert.Contains(t, conf.Settings, setting) +} + +func TestConfigurable_Configure(t *testing.T) { + setting := &Setting{ + EnvVariable: "TEST_SETTING", + Required: true, + } + conf := &Configurable{ + Settings: Settings{setting}, + Label: "TestConfigurable", + } + + _ = os.Setenv("TEST_SETTING", "test_value") + err := conf.Configure() + assert.NoError(t, err) + assert.Equal(t, "test_value", setting.Value) +} + +func TestConfigurable_Setup(t *testing.T) { + setting := &Setting{ + EnvVariable: "TEST_SETTING", + Required: false, + } + conf := &Configurable{ + Settings: Settings{setting}, + Label: "TestConfigurable", + } + + err := conf.Setup() + assert.NoError(t, err) +} + +func TestSetting_IsValid(t *testing.T) { + setting := &Setting{ + EnvVariable: "TEST_SETTING", + Value: "some_value", + Required: true, + } + + assert.True(t, setting.IsValid()) + + setting.Value = "" + assert.False(t, setting.IsValid()) +} + +func TestSetting_Configure(t *testing.T) { + _ = os.Setenv("TEST_SETTING", "test_value") + setting := &Setting{ + EnvVariable: "TEST_SETTING", + Required: true, + } + err := setting.Configure() + assert.NoError(t, err) + assert.Equal(t, "test_value", setting.Value) +} + +func TestSetting_FillEnvFileContent(t *testing.T) { + buffer := &bytes.Buffer{} + setting := &Setting{ + EnvVariable: "TEST_SETTING", + Value: "test_value", + } + setting.FillEnvFileContent(buffer) + + expected := "TEST_SETTING=test_value\n" + assert.Equal(t, expected, buffer.String()) +} + +func TestSetting_Print(t *testing.T) { + setting := &Setting{ + EnvVariable: "TEST_SETTING", + Value: "test_value", + } + expected := "TEST_SETTING: test_value\n" + fmtOutput := captureOutput(func() { + setting.Print() + }) + assert.Equal(t, expected, fmtOutput) +} + +func TestSetupQuestion_Ask(t *testing.T) { + setting := &Setting{ + EnvVariable: "TEST_SETTING", + Required: true, + } + question := &SetupQuestion{ + Setting: setting, + Question: "Enter test setting:", + } + input := "user_value\n" + fmtInput := captureInput(input) + defer fmtInput() + err := question.Ask("TestConfigurable") + assert.NoError(t, err) + assert.Equal(t, "user_value", setting.Value) +} + +func TestSettings_IsConfigured(t *testing.T) { + settings := Settings{ + {EnvVariable: "TEST_SETTING1", Value: "value1", Required: true}, + {EnvVariable: "TEST_SETTING2", Value: "", Required: false}, + } + + assert.True(t, settings.IsConfigured()) + + settings[0].Value = "" + assert.False(t, settings.IsConfigured()) +} + +func TestSettings_Configure(t *testing.T) { + _ = os.Setenv("TEST_SETTING", "test_value") + settings := Settings{ + {EnvVariable: "TEST_SETTING", Required: true}, + } + + err := settings.Configure() + assert.NoError(t, err) + assert.Equal(t, "test_value", settings[0].Value) +} + +func TestSettings_FillEnvFileContent(t *testing.T) { + buffer := &bytes.Buffer{} + settings := Settings{ + {EnvVariable: "TEST_SETTING", Value: "test_value"}, + } + settings.FillEnvFileContent(buffer) + + expected := "TEST_SETTING=test_value\n" + assert.Equal(t, expected, buffer.String()) +} + +// captureOutput captures the output of a function call +func captureOutput(f func()) string { + var buf bytes.Buffer + stdout := os.Stdout + r, w, _ := os.Pipe() + os.Stdout = w + f() + _ = w.Close() + os.Stdout = stdout + _, _ = buf.ReadFrom(r) + return buf.String() +} + +// captureInput captures the input for a function call +func captureInput(input string) func() { + r, w, _ := os.Pipe() + _, _ = w.WriteString(input) + _ = w.Close() + stdin := os.Stdin + os.Stdin = r + return func() { + os.Stdin = stdin + } +} diff --git a/common/domain.go b/common/domain.go index b11458c..f5a1e40 100644 --- a/common/domain.go +++ b/common/domain.go @@ -19,3 +19,24 @@ type ChatOptions struct { PresencePenalty float64 FrequencyPenalty float64 } + +// NormalizeMessages remove empty messages and ensure messages order user-assist-user +func NormalizeMessages(msgs []*Message, defaultUserMessage string) (ret []*Message) { + // Iterate over messages to enforce the odd position rule for user messages + fullMessageIndex := 0 + for _, message := range msgs { + if message.Content == "" { + // Skip empty messages as the anthropic API doesn't accept them + continue + } + + // Ensure, that each odd position shall be a user message + if fullMessageIndex%2 == 0 && message.Role != "user" { + ret = append(ret, &Message{Role: "user", Content: defaultUserMessage}) + fullMessageIndex++ + } + ret = append(ret, message) + fullMessageIndex++ + } + return +} diff --git a/common/domain_test.go b/common/domain_test.go new file mode 100644 index 0000000..a4b5ffe --- /dev/null +++ b/common/domain_test.go @@ -0,0 +1,25 @@ +package common + +import ( + "github.com/stretchr/testify/assert" + "testing" +) + +func TestNormalizeMessages(t *testing.T) { + msgs := []*Message{ + {Role: "user", Content: "Hello"}, + {Role: "bot", Content: "Hi there!"}, + {Role: "bot", Content: ""}, + {Role: "user", Content: ""}, + {Role: "user", Content: "How are you?"}, + } + + expected := []*Message{ + {Role: "user", Content: "Hello"}, + {Role: "bot", Content: "Hi there!"}, + {Role: "user", Content: "How are you?"}, + } + + actual := NormalizeMessages(msgs, "default") + assert.Equal(t, expected, actual) +} diff --git a/common/messages.go b/common/messages.go deleted file mode 100644 index dd1e330..0000000 --- a/common/messages.go +++ /dev/null @@ -1,22 +0,0 @@ -package common - -// NormalizeMessages remove empty messages and ensure messages order user-assist-user -func NormalizeMessages(msgs []*Message, defaultUserMessage string) (ret []*Message) { - // Iterate over messages to enforce the odd position rule for user messages - fullMessageIndex := 0 - for _, message := range msgs { - if message.Content == "" { - // Skip empty messages as the anthropic API doesn't accept them - continue - } - - // Ensure, that each odd position shall be a user message - if fullMessageIndex%2 == 0 && message.Role != "user" { - ret = append(ret, &Message{Role: "user", Content: defaultUserMessage}) - fullMessageIndex++ - } - ret = append(ret, message) - fullMessageIndex++ - } - return -} diff --git a/common/vendor.go b/common/vendor.go deleted file mode 100644 index d5c7aa0..0000000 --- a/common/vendor.go +++ /dev/null @@ -1,12 +0,0 @@ -package common - -type Vendor interface { - GetName() string - IsConfigured() bool - Configure() error - ListModels() ([]string, error) - SendStream([]*Message, *ChatOptions, chan string) error - Send([]*Message, *ChatOptions) (string, error) - GetSettings() Settings - Setup() error -} diff --git a/core/chatter.go b/core/chatter.go index f9c5c7f..70123f3 100644 --- a/core/chatter.go +++ b/core/chatter.go @@ -4,6 +4,7 @@ import ( "fmt" "github.com/danielmiessler/fabric/common" "github.com/danielmiessler/fabric/db" + "github.com/danielmiessler/fabric/vendors" ) type Chatter struct { @@ -12,7 +13,7 @@ type Chatter struct { Stream bool model string - vendor common.Vendor + vendor vendors.Vendor } func (o *Chatter) Send(request *common.ChatRequest, opts *common.ChatOptions) (message string, err error) { diff --git a/core/chatter_test.go b/core/chatter_test.go new file mode 100644 index 0000000..70966e7 --- /dev/null +++ b/core/chatter_test.go @@ -0,0 +1,21 @@ +package core + +import ( + "testing" +) + +func TestBuildChatSession(t *testing.T) { + chat := &Chat{ + Context: "test context", + Pattern: "test pattern", + Message: "test message", + } + session, err := chat.BuildChatSession() + if err != nil { + t.Fatalf("BuildChatSession() error = %v", err) + } + + if session == nil { + t.Fatalf("BuildChatSession() returned nil session") + } +} diff --git a/core/fabric.go b/core/fabric.go index a997256..7e295d2 100644 --- a/core/fabric.go +++ b/core/fabric.go @@ -9,7 +9,7 @@ import ( "github.com/danielmiessler/fabric/vendors/anthropic" "github.com/danielmiessler/fabric/vendors/azure" "github.com/danielmiessler/fabric/vendors/gemini" - "github.com/danielmiessler/fabric/vendors/grocq" + "github.com/danielmiessler/fabric/vendors/groc" "github.com/danielmiessler/fabric/vendors/ollama" "github.com/danielmiessler/fabric/vendors/openai" "github.com/danielmiessler/fabric/youtube" @@ -56,7 +56,7 @@ func NewFabricBase(db *db.Db) (ret *Fabric) { ret.DefaultModel = ret.AddSetupQuestionCustom("Model", true, "Enter the index the name of your default model") - ret.VendorsAll.AddVendors(openai.NewClient(), azure.NewClient(), ollama.NewClient(), grocq.NewClient(), + ret.VendorsAll.AddVendors(openai.NewClient(), azure.NewClient(), ollama.NewClient(), groc.NewClient(), gemini.NewClient(), anthropic.NewClient()) return @@ -85,13 +85,13 @@ func (o *Fabric) SaveEnvFile() (err error) { var envFileContent bytes.Buffer o.Settings.FillEnvFileContent(&envFileContent) - o.PatternsLoader.FillEnvFileContent(&envFileContent) + o.PatternsLoader.SetupFillEnvFileContent(&envFileContent) for _, vendor := range o.Vendors { - vendor.GetSettings().FillEnvFileContent(&envFileContent) + vendor.SetupFillEnvFileContent(&envFileContent) } - o.YouTube.FillEnvFileContent(&envFileContent) + o.YouTube.SetupFillEnvFileContent(&envFileContent) err = o.Db.SaveEnv(envFileContent.String()) return @@ -106,9 +106,7 @@ func (o *Fabric) Setup() (err error) { return } - if err = o.YouTube.Setup(); err != nil { - return - } + _ = o.YouTube.SetupOrSkip() if err = o.PatternsLoader.Setup(); err != nil { return @@ -152,16 +150,9 @@ func (o *Fabric) SetupDefaultModel() (err error) { } func (o *Fabric) SetupVendors() (err error) { - o.Reset() - - for _, vendor := range o.VendorsAll.Vendors { - fmt.Println() - if vendorErr := vendor.Setup(); vendorErr == nil { - fmt.Printf("[%v] configured\n", vendor.GetName()) - o.AddVendors(vendor) - } else { - fmt.Printf("[%v] skiped\n", vendor.GetName()) - } + o.Models = nil + if o.Vendors, err = o.VendorsAll.Setup(); err != nil { + return } if !o.HasVendors() { @@ -185,7 +176,8 @@ func (o *Fabric) configure() (err error) { return } - err = o.YouTube.Configure() + //YouTube is not mandatory, so ignore not configured error + _ = o.YouTube.Configure() return } diff --git a/core/fabric_test.go b/core/fabric_test.go new file mode 100644 index 0000000..cc2907a --- /dev/null +++ b/core/fabric_test.go @@ -0,0 +1,49 @@ +package core + +import ( + "os" + "testing" + + "github.com/danielmiessler/fabric/db" +) + +func TestNewFabric(t *testing.T) { + _, err := NewFabric(db.NewDb(os.TempDir())) + if err == nil { + t.Fatal("without setup error expected") + } +} + +func TestSaveEnvFile(t *testing.T) { + fabric := NewFabricBase(db.NewDb(os.TempDir())) + + err := fabric.SaveEnvFile() + if err != nil { + t.Fatalf("SaveEnvFile() error = %v", err) + } +} + +func TestCopyToClipboard(t *testing.T) { + t.Skip("skipping test, because of docker env. in ci.") + fabric := NewFabricBase(db.NewDb(os.TempDir())) + + message := "test message" + err := fabric.CopyToClipboard(message) + if err != nil { + t.Fatalf("CopyToClipboard() error = %v", err) + } +} + +func TestCreateOutputFile(t *testing.T) { + mockDb := &db.Db{} + fabric := NewFabricBase(mockDb) + + fileName := "test_output.txt" + message := "test message" + err := fabric.CreateOutputFile(message, fileName) + if err != nil { + t.Fatalf("CreateOutputFile() error = %v", err) + } + + defer os.Remove(fileName) +} diff --git a/core/models_test.go b/core/models_test.go new file mode 100644 index 0000000..addaa78 --- /dev/null +++ b/core/models_test.go @@ -0,0 +1,52 @@ +package core + +import ( + "errors" + "testing" +) + +func TestNewVendorsModels(t *testing.T) { + vendors := NewVendorsModels() + if vendors == nil { + t.Fatalf("NewVendorsModels() returned nil") + } + if len(vendors.VendorsModels) != 0 { + t.Fatalf("NewVendorsModels() returned non-empty VendorsModels map") + } +} + +func TestFindVendorsByModelFirst(t *testing.T) { + vendors := NewVendorsModels() + vendors.AddVendorModels("vendor1", []string{"model1", "model2"}) + vendor := vendors.FindVendorsByModelFirst("model1") + if vendor != "vendor1" { + t.Fatalf("FindVendorsByModelFirst() = %v, want %v", vendor, "vendor1") + } +} + +func TestFindVendorsByModel(t *testing.T) { + vendors := NewVendorsModels() + vendors.AddVendorModels("vendor1", []string{"model1", "model2"}) + foundVendors := vendors.FindVendorsByModel("model1") + if len(foundVendors) != 1 || foundVendors[0] != "vendor1" { + t.Fatalf("FindVendorsByModel() = %v, want %v", foundVendors, []string{"vendor1"}) + } +} + +func TestAddVendorModels(t *testing.T) { + vendors := NewVendorsModels() + vendors.AddVendorModels("vendor1", []string{"model1", "model2"}) + models := vendors.GetVendorModels("vendor1") + if len(models) != 2 { + t.Fatalf("AddVendorModels() failed to add models") + } +} + +func TestAddError(t *testing.T) { + vendors := NewVendorsModels() + err := errors.New("sample error") + vendors.AddError(err) + if len(vendors.Errs) != 1 { + t.Fatalf("AddError() failed to add error") + } +} diff --git a/core/vendors.go b/core/vendors.go index ec1629c..82f1a71 100644 --- a/core/vendors.go +++ b/core/vendors.go @@ -3,32 +3,27 @@ package core import ( "context" "fmt" - "github.com/danielmiessler/fabric/common" + "github.com/danielmiessler/fabric/vendors" "sync" ) func NewVendorsManager() *VendorsManager { return &VendorsManager{ - Vendors: map[string]common.Vendor{}, + Vendors: map[string]vendors.Vendor{}, } } type VendorsManager struct { - Vendors map[string]common.Vendor + Vendors map[string]vendors.Vendor Models *VendorsModels } -func (o *VendorsManager) AddVendors(vendors ...common.Vendor) { +func (o *VendorsManager) AddVendors(vendors ...vendors.Vendor) { for _, vendor := range vendors { o.Vendors[vendor.GetName()] = vendor } } -func (o *VendorsManager) Reset() { - o.Vendors = map[string]common.Vendor{} - o.Models = nil -} - func (o *VendorsManager) GetModels() *VendorsModels { if o.Models == nil { o.readModels() @@ -40,7 +35,7 @@ func (o *VendorsManager) HasVendors() bool { return len(o.Vendors) > 0 } -func (o *VendorsManager) FindByName(name string) common.Vendor { +func (o *VendorsManager) FindByName(name string) vendors.Vendor { return o.Vendors[name] } @@ -76,7 +71,7 @@ func (o *VendorsManager) readModels() { } func (o *VendorsManager) fetchVendorModels( - ctx context.Context, wg *sync.WaitGroup, vendor common.Vendor, resultsChan chan<- modelResult) { + ctx context.Context, wg *sync.WaitGroup, vendor vendors.Vendor, resultsChan chan<- modelResult) { defer wg.Done() @@ -90,6 +85,20 @@ func (o *VendorsManager) fetchVendorModels( } } +func (o *VendorsManager) Setup() (ret map[string]vendors.Vendor, err error) { + ret = map[string]vendors.Vendor{} + for _, vendor := range o.Vendors { + fmt.Println() + if vendorErr := vendor.Setup(); vendorErr == nil { + fmt.Printf("[%v] configured\n", vendor.GetName()) + ret[vendor.GetName()] = vendor + } else { + fmt.Printf("[%v] skipped\n", vendor.GetName()) + } + } + return +} + type modelResult struct { vendorName string models []string diff --git a/core/vendors_test.go b/core/vendors_test.go new file mode 100644 index 0000000..17063de --- /dev/null +++ b/core/vendors_test.go @@ -0,0 +1,129 @@ +package core + +import ( + "bytes" + "github.com/danielmiessler/fabric/common" + "testing" +) + +func TestNewVendorsManager(t *testing.T) { + vendorsManager := NewVendorsManager() + if vendorsManager == nil { + t.Fatalf("NewVendorsManager() returned nil") + } +} + +func TestAddVendors(t *testing.T) { + vendorsManager := NewVendorsManager() + mockVendor := &MockVendor{name: "testVendor"} + vendorsManager.AddVendors(mockVendor) + + if _, exists := vendorsManager.Vendors[mockVendor.GetName()]; !exists { + t.Fatalf("AddVendors() did not add vendor") + } +} + +func TestGetModels(t *testing.T) { + vendorsManager := NewVendorsManager() + mockVendor := &MockVendor{name: "testVendor"} + vendorsManager.AddVendors(mockVendor) + + models := vendorsManager.GetModels() + if models == nil { + t.Fatalf("GetModels() returned nil") + } +} + +func TestHasVendors(t *testing.T) { + vendorsManager := NewVendorsManager() + if vendorsManager.HasVendors() { + t.Fatalf("HasVendors() should return false for an empty manager") + } + + mockVendor := &MockVendor{name: "testVendor"} + vendorsManager.AddVendors(mockVendor) + if !vendorsManager.HasVendors() { + t.Fatalf("HasVendors() should return true after adding a vendor") + } +} + +func TestFindByName(t *testing.T) { + vendorsManager := NewVendorsManager() + mockVendor := &MockVendor{name: "testVendor"} + vendorsManager.AddVendors(mockVendor) + + foundVendor := vendorsManager.FindByName("testVendor") + if foundVendor == nil { + t.Fatalf("FindByName() did not find added vendor") + } +} + +func TestReadModels(t *testing.T) { + vendorsManager := NewVendorsManager() + mockVendor := &MockVendor{name: "testVendor"} + vendorsManager.AddVendors(mockVendor) + + vendorsManager.readModels() + if vendorsManager.Models == nil || len(vendorsManager.Models.Vendors) == 0 { + t.Fatalf("readModels() did not read models correctly") + } +} + +func TestSetup(t *testing.T) { + vendorsManager := NewVendorsManager() + mockVendor := &MockVendor{name: "testVendor"} + vendorsManager.AddVendors(mockVendor) + + vendors, err := vendorsManager.Setup() + if err != nil { + t.Fatalf("Setup() error = %v", err) + } + if len(vendors) == 0 { + t.Fatalf("Setup() did not setup any vendors") + } +} + +// MockVendor is a mock implementation of the Vendor interface for testing purposes. +type MockVendor struct { + *common.Settings + name string +} + +func (o *MockVendor) SendStream(messages []*common.Message, options *common.ChatOptions, strings chan string) error { + //TODO implement me + panic("implement me") +} + +func (o *MockVendor) Send(messages []*common.Message, options *common.ChatOptions) (string, error) { + //TODO implement me + panic("implement me") +} + +func (o *MockVendor) SetupFillEnvFileContent(buffer *bytes.Buffer) { + //TODO implement me + panic("implement me") +} + +func (o *MockVendor) IsConfigured() bool { + return false +} + +func (o *MockVendor) GetSettings() *common.Settings { + return o.Settings +} + +func (o *MockVendor) GetName() string { + return o.name +} + +func (o *MockVendor) Configure() error { + return nil +} + +func (o *MockVendor) Setup() error { + return nil +} + +func (o *MockVendor) ListModels() ([]string, error) { + return []string{"model1", "model2"}, nil +} diff --git a/db/contexts_test.go b/db/contexts_test.go new file mode 100644 index 0000000..678d6f8 --- /dev/null +++ b/db/contexts_test.go @@ -0,0 +1,29 @@ +package db + +import ( + "os" + "path/filepath" + "testing" +) + +func TestContexts_GetContext(t *testing.T) { + dir := t.TempDir() + contexts := &Contexts{ + Storage: &Storage{Dir: dir}, + } + contextName := "testContext" + contextPath := filepath.Join(dir, contextName) + contextContent := "test content" + err := os.WriteFile(contextPath, []byte(contextContent), 0644) + if err != nil { + t.Fatalf("failed to write context file: %v", err) + } + context, err := contexts.GetContext(contextName) + if err != nil { + t.Fatalf("failed to get context: %v", err) + } + expectedContext := &Context{Name: contextName, Content: contextContent} + if *context != *expectedContext { + t.Errorf("expected %v, got %v", expectedContext, context) + } +} diff --git a/db/db_test.go b/db/db_test.go new file mode 100644 index 0000000..53135cf --- /dev/null +++ b/db/db_test.go @@ -0,0 +1,55 @@ +package db + +import ( + "os" + "testing" +) + +func TestDb_Configure(t *testing.T) { + dir := t.TempDir() + db := NewDb(dir) + err := db.Configure() + if err == nil { + t.Fatalf("db is configured, but must not be at empty dir: %v", dir) + } + if db.IsEnvFileExists() { + t.Fatalf("db file exists, but must not be at empty dir: %v", dir) + } + + err = db.SaveEnv("") + if err != nil { + t.Fatalf("db can't save env for empty conf.: %v", err) + } + + err = db.Configure() + if err != nil { + t.Fatalf("db is not configured, but shall be after save: %v", err) + } +} + +func TestDb_LoadEnvFile(t *testing.T) { + dir := t.TempDir() + db := NewDb(dir) + content := "KEY=VALUE\n" + err := os.WriteFile(db.EnvFilePath, []byte(content), 0644) + if err != nil { + t.Fatalf("failed to write .env file: %v", err) + } + err = db.LoadEnvFile() + if err != nil { + t.Errorf("failed to load .env file: %v", err) + } +} + +func TestDb_SaveEnv(t *testing.T) { + dir := t.TempDir() + db := NewDb(dir) + content := "KEY=VALUE\n" + err := db.SaveEnv(content) + if err != nil { + t.Errorf("failed to save .env file: %v", err) + } + if _, err := os.Stat(db.EnvFilePath); os.IsNotExist(err) { + t.Errorf("expected .env file to be saved") + } +} diff --git a/db/patterns_test.go b/db/patterns_test.go new file mode 100644 index 0000000..3a49c63 --- /dev/null +++ b/db/patterns_test.go @@ -0,0 +1 @@ +package db diff --git a/db/sessions_test.go b/db/sessions_test.go new file mode 100644 index 0000000..c420a4a --- /dev/null +++ b/db/sessions_test.go @@ -0,0 +1,38 @@ +package db + +import ( + "testing" + + "github.com/danielmiessler/fabric/common" +) + +func TestSessions_GetOrCreateSession(t *testing.T) { + dir := t.TempDir() + sessions := &Sessions{ + Storage: &Storage{Dir: dir, FileExtension: ".json"}, + } + sessionName := "testSession" + session, err := sessions.GetOrCreateSession(sessionName) + if err != nil { + t.Fatalf("failed to get or create session: %v", err) + } + if session.Name != sessionName { + t.Errorf("expected session name %v, got %v", sessionName, session.Name) + } +} + +func TestSessions_SaveSession(t *testing.T) { + dir := t.TempDir() + sessions := &Sessions{ + Storage: &Storage{Dir: dir, FileExtension: ".json"}, + } + sessionName := "testSession" + session := &Session{Name: sessionName, Messages: []*common.Message{{Content: "message1"}}} + err := sessions.SaveSession(session) + if err != nil { + t.Fatalf("failed to save session: %v", err) + } + if !sessions.Exists(sessionName) { + t.Errorf("expected session to be saved") + } +} diff --git a/db/storage_test.go b/db/storage_test.go new file mode 100644 index 0000000..529c441 --- /dev/null +++ b/db/storage_test.go @@ -0,0 +1,52 @@ +package db + +import ( + "testing" +) + +func TestStorage_SaveAndLoad(t *testing.T) { + dir := t.TempDir() + storage := &Storage{Dir: dir} + name := "test" + content := []byte("test content") + if err := storage.Save(name, content); err != nil { + t.Fatalf("failed to save content: %v", err) + } + loadedContent, err := storage.Load(name) + if err != nil { + t.Fatalf("failed to load content: %v", err) + } + if string(loadedContent) != string(content) { + t.Errorf("expected %v, got %v", string(content), string(loadedContent)) + } +} + +func TestStorage_Exists(t *testing.T) { + dir := t.TempDir() + storage := &Storage{Dir: dir} + name := "test" + if storage.Exists(name) { + t.Errorf("expected file to not exist") + } + if err := storage.Save(name, []byte("test content")); err != nil { + t.Fatalf("failed to save content: %v", err) + } + if !storage.Exists(name) { + t.Errorf("expected file to exist") + } +} + +func TestStorage_Delete(t *testing.T) { + dir := t.TempDir() + storage := &Storage{Dir: dir} + name := "test" + if err := storage.Save(name, []byte("test content")); err != nil { + t.Fatalf("failed to save content: %v", err) + } + if err := storage.Delete(name); err != nil { + t.Fatalf("failed to delete content: %v", err) + } + if storage.Exists(name) { + t.Errorf("expected file to be deleted") + } +} diff --git a/go.mod b/go.mod index 086de6e..d86e0fa 100644 --- a/go.mod +++ b/go.mod @@ -16,8 +16,8 @@ require ( github.com/pkg/errors v0.9.1 github.com/samber/lo v1.47.0 github.com/sashabaranov/go-openai v1.28.2 + github.com/stretchr/testify v1.9.0 google.golang.org/api v0.192.0 - gopkg.in/gookit/color.v1 v1.1.6 ) require ( @@ -30,8 +30,10 @@ require ( dario.cat/mergo v1.0.0 // indirect github.com/Microsoft/go-winio v0.6.1 // indirect github.com/ProtonMail/go-crypto v1.0.0 // indirect + github.com/anaskhan96/soup v1.2.5 // indirect github.com/cloudflare/circl v1.3.7 // indirect github.com/cyphar/filepath-securejoin v0.2.4 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect github.com/emirpasic/gods v1.18.1 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect @@ -46,6 +48,7 @@ require ( github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect github.com/kevinburke/ssh_config v1.2.0 // indirect github.com/pjbgf/sha1cd v0.3.0 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect github.com/skeema/knownhosts v1.2.2 // indirect github.com/xanzy/ssh-agent v0.3.3 // indirect @@ -69,4 +72,5 @@ require ( google.golang.org/grpc v1.64.1 // indirect google.golang.org/protobuf v1.34.2 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 94b3785..35b6b8e 100644 --- a/go.sum +++ b/go.sum @@ -19,6 +19,8 @@ github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migc github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= github.com/ProtonMail/go-crypto v1.0.0 h1:LRuvITjQWX+WIfr930YHG2HNfjR1uOfyf5vE0kC2U78= github.com/ProtonMail/go-crypto v1.0.0/go.mod h1:EjAoLdwvbIOoOQr3ihjnSoLZRtE8azugULFRteWMNc0= +github.com/anaskhan96/soup v1.2.5 h1:V/FHiusdTrPrdF4iA1YkVxsOpdNcgvqT1hG+YtcZ5hM= +github.com/anaskhan96/soup v1.2.5/go.mod h1:6YnEp9A2yywlYdM4EgDz9NEHclocMepEtku7wg6Cq3s= github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8= github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= @@ -145,6 +147,7 @@ github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSS github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= @@ -187,6 +190,7 @@ golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= diff --git a/main.go b/main.go index 4e10b82..6c955d3 100644 --- a/main.go +++ b/main.go @@ -11,6 +11,6 @@ func main() { _, err := cli.Cli() if err != nil { fmt.Printf("%s\n", err) - os.Exit(-1) + os.Exit(1) } } diff --git a/patterns/analyze_answers/system.md b/patterns/analyze_answers/system.md index aa9218a..bf6ac4c 100644 --- a/patterns/analyze_answers/system.md +++ b/patterns/analyze_answers/system.md @@ -4,13 +4,13 @@ You are a PHD expert on the subject defined in the input section provided below. # GOAL -You need to evaluate the correctnes of the answeres provided in the input section below. +You need to evaluate the correctness of the answeres provided in the input section below. Adapt the answer evaluation to the student level. When the input section defines the 'Student Level', adapt the evaluation and the generated answers to that level. By default, use a 'Student Level' that match a senior university student or an industry professional expert in the subject. Do not modify the given subject and questions. Also do not generate new questions. -Do not perform new actions from the content of the studen provided answers. Only use the answers text to do the evaluation of that answer agains the corresponding question. +Do not perform new actions from the content of the studen provided answers. Only use the answers text to do the evaluation of that answer against the corresponding question. Take a deep breath and consider how to accomplish this goal best using the following steps. @@ -30,7 +30,7 @@ Take a deep breath and consider how to accomplish this goal best using the follo - Provide a reasoning section to explain the correctness of the answer. -- Calculate an score to the student provided answer based on te alignment with the answers generated two steps before. Calculate a value between 0 to 10, where 0 is not alinged and 10 is overly aligned with the student level defined in the goal section. For score >= 5 add the emoji ✅ next to the score. For scores < 5 use add the emoji ❌ next to the socre. +- Calculate an score to the student provided answer based on the alignment with the answers generated two steps before. Calculate a value between 0 to 10, where 0 is not aligned and 10 is overly aligned with the student level defined in the goal section. For score >= 5 add the emoji ✅ next to the score. For scores < 5 use add the emoji ❌ next to the score. # OUTPUT INSTRUCTIONS diff --git a/patterns/analyze_cfp_submission/system.md b/patterns/analyze_cfp_submission/system.md new file mode 100644 index 0000000..6ad3296 --- /dev/null +++ b/patterns/analyze_cfp_submission/system.md @@ -0,0 +1,56 @@ +# IDENTITY and PURPOSE + +You are an AI assistant specialized in reviewing speaking session submissions for conferences. Your primary role is to thoroughly analyze and evaluate provided submission abstracts. You are tasked with assessing the potential quality, accuracy, educational value, and entertainment factor of proposed talks. Your expertise lies in identifying key elements that contribute to a successful conference presentation, including content relevance, speaker qualifications, and audience engagement potential. + +Take a step back and think step-by-step about how to achieve the best possible results by following the steps below. + +# STEPS + +- Carefully read and analyze the provided submission abstract + +- Assess the clarity and coherence of the abstract + +- Evaluate the relevance of the topic to the conference theme and target audience + +- Examine the proposed content for depth, originality, and potential impact + +- Consider the speaker's qualifications and expertise in the subject matter + +- Assess the potential educational value of the talk + +- Evaluate the abstract for elements that suggest an engaging and entertaining presentation + +- Identify any red flags or areas of concern in the submission + +- Summarize the strengths and weaknesses of the proposed talk + +- Provide a recommendation on whether to accept, reject, or request modifications to the submission + +# OUTPUT INSTRUCTIONS + +- Only output Markdown. + +- Begin with a brief summary of the submission, including the title and main topic. + +- Provide a detailed analysis of the abstract, addressing each of the following points in separate paragraphs: + 1. Clarity and coherence + 2. Relevance to conference and audience + 3. Content depth and originality + 4. Speaker qualifications + 5. Educational value + 6. Entertainment potential + 7. Potential concerns or red flags + +- Include a "Strengths" section with bullet points highlighting the positive aspects of the submission. + +- Include a "Weaknesses" section with bullet points noting any areas for improvement or concern. + +- Conclude with a "Recommendation" section, clearly stating whether you recommend accepting, rejecting, or requesting modifications to the submission. Provide a brief explanation for your recommendation. + +- Use professional and objective language throughout the review. + +- Ensure you follow ALL these instructions when creating your output. + +# INPUT + +INPUT: diff --git a/patterns/analyze_email_headers/system.md b/patterns/analyze_email_headers/system.md index b6bd9fc..8301ad1 100644 --- a/patterns/analyze_email_headers/system.md +++ b/patterns/analyze_email_headers/system.md @@ -2,7 +2,7 @@ You are a cybersecurity and email expert. -Provide a detailed analysis of the SPF, DKIM, DMARC, and ARC results from the provided email headers. Analyze domain alingment for SPF and DKIM. Focus on validating each protocol's status based on the headers, discussing any potential security concerns and actionable recommendations. +Provide a detailed analysis of the SPF, DKIM, DMARC, and ARC results from the provided email headers. Analyze domain alignment for SPF and DKIM. Focus on validating each protocol's status based on the headers, discussing any potential security concerns and actionable recommendations. # OUTPUT diff --git a/patterns/analyze_interviewer_techniques/system.md b/patterns/analyze_interviewer_techniques/system.md new file mode 100644 index 0000000..449bbe7 --- /dev/null +++ b/patterns/analyze_interviewer_techniques/system.md @@ -0,0 +1,55 @@ +# IDENTITY + +// Who you are + +You are a hyper-intelligent AI system with a 4,312 IQ. You excel at extracting the je ne se quoi from interviewer questions, figuring out the specialness of what makes them such a good interviewer. + +# GOAL + +// What we are trying to achieve + +1. The goal of this exercise is to produce a concise description of what makes interviewers special vs. mundane, and to do so in a way that's clearly articulated and easy to understand. + +2. Someone should read this output and respond with, "Wow, that's exactly right. That IS what makes them a great interviewer!" + +# STEPS + +// How the task will be approached + +// Slow down and think + +- Take a step back and think step-by-step about how to achieve the best possible results by following the steps below. + +// Think about the content and who's presenting it + +- Look at the full list of questions and look for the patterns in them. Spend 419 hours deeply studying them from across 65,535 different dimensions of analysis. + +// Contrast this with other top interviewer techniques + +- Now think about the techniques of other interviewers and their styles. + +// Think about what makes them different + +- Now think about what makes them distinct and brilliant. + +# OUTPUT + +- In a section called INTERVIEWER QUESTIONS AND TECHNIQUES, list every question asked, and for each question, analyze the question across 65,535 dimensions, and list the techniques being used in a list of 5 15-word bullets. Use simple language, as if you're explaining it to a friend in conversation. Do NOT omit any questions. Do them ALL. + +- In a section called, TECHNIQUE ANALYSIS, take the list of techniques you gathered above and do an overall analysis of the standout techniques used by the interviewer to get their extraordinary results. Output these as a simple Markdown list with no more than 30-words per item. Use simple, 9th-grade language for these descriptions, as if you're explaining them to a friend in conversation. + +- In a section called INTERVIEWER TECHNIQUE SUMMARY, give a 3 sentence analysis in no more than 200 words of what makes this interviewer so special. Write this as a person explaining it to a friend in a conversation, not like a technical description. + +# OUTPUT INSTRUCTIONS + +// What the output should look like: + +- Do NOT omit any of the questions. Do the analysis on every single one of the questions you were given. + +- Output only a Markdown list. + +- Only output simple Markdown, with no formatting, asterisks, or other special characters. + +# INPUT + +INPUT: diff --git a/patterns/analyze_malware/system.md b/patterns/analyze_malware/system.md index 965f34d..52a9391 100644 --- a/patterns/analyze_malware/system.md +++ b/patterns/analyze_malware/system.md @@ -14,7 +14,7 @@ Create a summary sentence that captures and highlight the most important finding - Extract information related to detection in a section called DETECTION. - Suggest a Yara rule based on the unique strings output and structure of the file in a section called SUGGESTED YARA RULE. - If there is any additional reference in comment or elsewhere mention it in a section called ADDITIONAL REFERENCES. -- Provide some recommandation in term of detection and further steps only backed by technical data you have in a section called RECOMMANDATIONS. +- Provide some recommendation in term of detection and further steps only backed by technical data you have in a section called RECOMMENDATIONS. # OUTPUT INSTRUCTIONS Only output Markdown. diff --git a/patterns/clean_text/system.md b/patterns/clean_text/system.md index 6b642be..cb38808 100644 --- a/patterns/clean_text/system.md +++ b/patterns/clean_text/system.md @@ -6,7 +6,7 @@ You are an expert at cleaning up broken and, malformatted, text, for example: li - Read the entire document and fully understand it. - Remove any strange line breaks that disrupt formatting. -- Add captialization, punctuation, line breaks, paragraphs and other formatting where necessary. +- Add capitalization, punctuation, line breaks, paragraphs and other formatting where necessary. - Do NOT change any content or spelling whatsoever. # OUTPUT INSTRUCTIONS diff --git a/patterns/create_git_diff_commit/system.md b/patterns/create_git_diff_commit/system.md index 813d734..cab57bc 100644 --- a/patterns/create_git_diff_commit/system.md +++ b/patterns/create_git_diff_commit/system.md @@ -6,7 +6,7 @@ You are an expert project manager and developer, and you specialize in creating - Read the input and figure out what the major changes and upgrades were that happened. -- Create the git commands needed to add the changes to the repo, and a git commit to reflet the changes +- Create the git commands needed to add the changes to the repo, and a git commit to reflect the changes - If there are a lot of changes include more bullets. If there are only a few changes, be more terse. diff --git a/patterns/create_report_finding/system.md b/patterns/create_report_finding/system.md index 5ba0a38..06f8d32 100644 --- a/patterns/create_report_finding/system.md +++ b/patterns/create_report_finding/system.md @@ -18,7 +18,7 @@ Take a step back and think step-by-step about how to achieve the best possible r - Extract the 5 to 15 of the most surprising, insightful, and/or interesting recommendations that can be collected from the report into a section called Recommendations. -- Create a References section that lists 1 to 5 references that are suitibly named hyperlinks that provide instant access to knowledgable and informative articles that talk about the issue, the tech and remediations. Do not hallucinate or act confident if you are unsure. +- Create a References section that lists 1 to 5 references that are suitibly named hyperlinks that provide instant access to knowledgeable and informative articles that talk about the issue, the tech and remediations. Do not hallucinate or act confident if you are unsure. - Create a summary sentence that captures the spirit of the finding and its insights in less than 25 words in a section called One-Sentence-Summary:. Use plain and conversational language when creating this summary. Don't use jargon or marketing language. diff --git a/patterns/create_threat_scenarios/system.md b/patterns/create_threat_scenarios/system.md index 786c6a5..b7c6342 100644 --- a/patterns/create_threat_scenarios/system.md +++ b/patterns/create_threat_scenarios/system.md @@ -53,7 +53,7 @@ It is not some stealth technology that makes you invisible online, because if in Now, let’s look at who we’re defending against if you use a VPN. Your ISP. If your VPN includes all DNS requests and traffic then you could be hiding significantly from your ISP. This is true. They’d still see traffic amounts, and there are some technologies that allow people to infer the contents of encrypted connections, but in general this is a good control if you’re worried about your ISP. The Government. If the government investigates you by only looking at your ISP, and you’ve been using your VPN 24-7, you’ll be in decent shape because it’ll just be encrypted traffic to a VPN provider. But now they’ll know that whatever you were doing was sensitive enough to use a VPN at all times. So, probably not a win. Besides, they’ll likely be looking at the places you’re actually visiting as well (the sites you’re going to on the VPN), and like I talked about above, that’s when your cloaking device is useless. You have to de-cloak to fire, basically. -Super Hackers Trying to Hack You. First, I don’t know who these super hackers are, or why they’re trying ot hack you. But if it’s a state-level hacking group (or similar elite level), and you are targeted, you’re going to get hacked unless you stop using the internet and email. It’s that simple. There are too many vulnerabilities in all systems, and these teams are too good, for you to be able to resist for long. You will eventually be hacked via phishing, social engineering, poisoning a site you already frequent, or some other technique. Focus instead on not being targeted. +Super Hackers Trying to Hack You. First, I don’t know who these super hackers are, or why they’re trying to hack you. But if it’s a state-level hacking group (or similar elite level), and you are targeted, you’re going to get hacked unless you stop using the internet and email. It’s that simple. There are too many vulnerabilities in all systems, and these teams are too good, for you to be able to resist for long. You will eventually be hacked via phishing, social engineering, poisoning a site you already frequent, or some other technique. Focus instead on not being targeted. Script Kiddies. If you are just trying to avoid general hacker-types trying to hack you, well, I don’t even know what that means. Again, the main advantage you get from a VPN is obscuring your traffic from your ISP. So unless this script kiddie had access to your ISP and nothing else, this doesn’t make a ton of sense. Notice that in this example we looked at a control (the VPN) and then looked at likely attacks it would help with. This is the opposite of looking at the attacks (like in the house scenario) and then thinking about controls. Using Everyday Threat Modeling includes being able to do both. Example 3: Using Smart Speakers in the House diff --git a/patterns/extract_questions/system.md b/patterns/extract_questions/system.md index 60aad42..4ea1937 100644 --- a/patterns/extract_questions/system.md +++ b/patterns/extract_questions/system.md @@ -1,18 +1,27 @@ # IDENTITY -You are an advanced AI with a 419 IQ that excels at asking brilliant questions of people. You specialize in extracting the questions out of a piece of content, word for word, and then figuring out what made the questions so good. +You are an advanced AI with a 419 IQ that excels at extracting all of the questions asked by an interviewer within a conversation. # GOAL -- Extract all the questions from the content. +- Extract all the questions asked by an interviewer in the input. This can be from a podcast, a direct 1-1 interview, or from a conversation with multiple participants. -- Determine what made the questions so good at getting surprising and high-quality answers from the person being asked. +- Ensure you get them word for word, because that matters. + +# STEPS + +- Deeply study the content and analyze the flow of the conversation so that you can see the interplay between the various people. This will help you determine who the interviewer is and who is being interviewed. + +- Extract all the questions asked by the interviewer. # OUTPUT -- In a section called QUESTIONS, list all questions as a series of bullet points. +- In a section called QUESTIONS, list all questions by the interviewer listed as a series of bullet points. -- In a section called ANALYSIS, give a set 15-word bullet points that capture the genius of the questions that were asked. +# OUTPUT INSTRUCTIONS -- In a section called RECOMMENDATIONS FOR INTERVIEWERS, give a set of 15-word bullet points that give prescriptive advice to interviewers on how to ask questions. +- Only output the list of questions asked by the interviewer. Don't add analysis or commentary or anything else. Just the questions. +- Output the list in a simple bulleted Markdown list. No formatting—just the list of questions. + +- Don't miss any questions. Do your analysis 1124 times to make sure you got them all. diff --git a/patterns/extract_wisdom_dm/system.md b/patterns/extract_wisdom_dm/system.md index d352c48..0224c83 100644 --- a/patterns/extract_wisdom_dm/system.md +++ b/patterns/extract_wisdom_dm/system.md @@ -26,23 +26,6 @@ You are a hyper-intelligent AI system with a 4,312 IQ. You excel at extracting i // Think about the ideas -- Extract ALL interesting points made in the content by any participant into a section called POINTS. Capture the point as 15-25 word bullet point. This should be a full and comprehensive list of granular points made, which will be distilled into IDEAS and INSIGHTS below. - -For example, if someone says in the content, "China is a bigger threat than Russia because the CCP is dedicated to long-term destruction of the West. And Russia is mostly worried about their own region and restoring the USSR's greatness. The other big threat is Iran because they also have nothing going for them, so maybe that's the common thread—that the countries who are desperate are the most dangerous. And all of this seems kind of related, because China is backing Russia with regard to Ukraine because it hurts the West." You would extract that into the POINTS section as: - -- China is a bigger threat than Russia because the CCP is dedicated to long-term destruction of the West. -- Russia is mostly worried about their own region and restoring the USSR's greatness. -- Iran is a big threat because they have nothing going for them. -- The common thread is that desperate countries are the most dangerous. -- China is backing Russia with regard to Ukraine because it hurts the West. -- Which means all of this is largely intertwined. - -Do that kind of extraction for all points made in the content. Again, ALL points. - -Organize these into 2-3 word sub-sections that indicate the topic, e.g., "AI", "The Ukraine War", "Continuous Learning", "Reading", etc. Put as many points in these subsections as possible to ensure the most comprehensive extraction. Don't worry about having a set number in each. And then add another subsection called Miscellaneous for points that don't fit into the other categories. DO NOT omit any interesting points made. - -- Make sure you extract at least 50 points into the POINTS section. - - Extract 20 to 50 of the most surprising, insightful, and/or interesting ideas from the input in a section called IDEAS:. If there are less than 50 then collect all of them. Make sure you extract at least 20. // Think about the insights that come from those ideas diff --git a/patterns/improve_report_finding/system.md b/patterns/improve_report_finding/system.md index 597ff9e..0529ffb 100644 --- a/patterns/improve_report_finding/system.md +++ b/patterns/improve_report_finding/system.md @@ -16,7 +16,7 @@ Take a step back and think step-by-step about how to achieve the best possible r - Extract the 5 to 15 of the most surprising, insightful, and/or interesting recommendations that can be collected from the report into a section called Recommendations. -- Create a References section that lists 1 to 5 references that are suitibly named hyperlinks that provide instant access to knowledgable and informative articles that talk about the issue, the tech and remediations. Do not hallucinate or act confident if you are unsure. +- Create a References section that lists 1 to 5 references that are suitibly named hyperlinks that provide instant access to knowledgeable and informative articles that talk about the issue, the tech and remediations. Do not hallucinate or act confident if you are unsure. - Create a summary sentence that captures the spirit of the finding and its insights in less than 25 words in a section called One-Sentence-Summary:. Use plain and conversational language when creating this summary. Don't use jargon or marketing language. diff --git a/patterns/show_fabric_options_markmap/system.md b/patterns/show_fabric_options_markmap/system.md index 81de699..3696931 100644 --- a/patterns/show_fabric_options_markmap/system.md +++ b/patterns/show_fabric_options_markmap/system.md @@ -171,7 +171,7 @@ us the results in --remoteOllamaServer REMOTEOLLAMASERVER The URL of the remote ollamaserver to use. ONLY USE THIS if you are using a local ollama server in an non- - deault location or port + default location or port --context, -c Use Context file (context.md) to add context to your pattern age: fabric [-h] [--text TEXT] [--copy] [--agents {trip_planner,ApiKeys}] diff --git a/patterns/write_hackerone_report/system.md b/patterns/write_hackerone_report/system.md index 7d26d17..1913ccf 100644 --- a/patterns/write_hackerone_report/system.md +++ b/patterns/write_hackerone_report/system.md @@ -1,6 +1,6 @@ # IDENTITY -You are an exceptionally talented bug bounty hunter that specializes in writing bug bounty reports that are concise, to-the-point, and easy to reproduce. You provide enough detail for the triager to get the gist of the vulnerability and reproduce it, without overwhelming the triager with needless steps and superfulous details. +You are an exceptionally talented bug bounty hunter that specializes in writing bug bounty reports that are concise, to-the-point, and easy to reproduce. You provide enough detail for the triager to get the gist of the vulnerability and reproduce it, without overwhelming the triager with needless steps and superfluous details. # GOALS diff --git a/patterns/write_nuclei_template_rule/system.md b/patterns/write_nuclei_template_rule/system.md index 537c7d0..9a877bb 100644 --- a/patterns/write_nuclei_template_rule/system.md +++ b/patterns/write_nuclei_template_rule/system.md @@ -1708,7 +1708,7 @@ log(uniq.Values()) ``` And that’s it, this automatically converts any slice/array to map and removes duplicates from it and returns a slice/array of unique values -Similar to DSL helper functions . we can either use built in functions available with Javscript (ECMAScript 5.1) or use DSL helper functions and its upto user to decide which one to uses. +Similar to DSL helper functions . we can either use built in functions available with Javascript (ECMAScript 5.1) or use DSL helper functions and its upto user to decide which one to uses. ``` - method: GET # http request @@ -1733,7 +1733,7 @@ Important Matcher Rules: - Just like the XSS templates SSRF template also results in False Positives so make sure to add additional matcher from the response to the template. We have seen honeypots sending request to any URL they may receive in GET/POST data which will result in FP if we are just using the HTTP/DNS interactsh matcher. - For Time-based SQL Injection templates, if we must have to add duration dsl for the detection, make sure to add additional string from the vulnerable endpoint to avoid any FP that can be due to network error. -Make sure there are no yaml erros in a valid nuclei templates like the following +Make sure there are no yaml errors in a valid nuclei templates like the following - trailing spaces - wrong indentation errosr like: expected 10 but found 9 diff --git a/utils/log.go b/utils/log.go deleted file mode 100644 index c100f17..0000000 --- a/utils/log.go +++ /dev/null @@ -1,28 +0,0 @@ -package utils - -import ( - "fmt" - "os" - - "gopkg.in/gookit/color.v1" -) - -func Print(info string) { - fmt.Println(info) -} - -func PrintWarning (s string) { - fmt.Println(color.Yellow.Render("Warning: " + s)) -} - -func LogError(err error) { - fmt.Fprintln(os.Stderr, color.Red.Render(err.Error())) -} - -func LogWarning(err error) { - fmt.Fprintln(os.Stderr, color.Yellow.Render(err.Error())) -} - -func Log(info string) { - fmt.Println(color.Green.Render(info)) -} \ No newline at end of file diff --git a/vendors/azure/azure.go b/vendors/azure/azure.go index c4f6c4f..a336a41 100644 --- a/vendors/azure/azure.go +++ b/vendors/azure/azure.go @@ -10,9 +10,7 @@ import ( func NewClient() (ret *Client) { ret = &Client{} - ret.Client = openai.NewClientCompatible("Azure", ret.configure) - - ret.ApiEndpoint = ret.AddSetupQuestion("API endpoint", true) + ret.Client = openai.NewClientCompatible("Azure", "", ret.configure) ret.ApiDeployments = ret.AddSetupQuestionCustom("deployments", true, "Enter your Azure deployments (comma separated)") @@ -21,7 +19,6 @@ func NewClient() (ret *Client) { type Client struct { *openai.Client - ApiEndpoint *common.SetupQuestion ApiDeployments *common.SetupQuestion apiDeployments []string @@ -29,7 +26,7 @@ type Client struct { func (oi *Client) configure() (err error) { oi.apiDeployments = strings.Split(oi.ApiDeployments.Value, ",") - oi.ApiClient = goopenai.NewClientWithConfig(goopenai.DefaultAzureConfig(oi.ApiKey.Value, oi.ApiEndpoint.Value)) + oi.ApiClient = goopenai.NewClientWithConfig(goopenai.DefaultAzureConfig(oi.ApiKey.Value, oi.ApiBaseURL.Value)) return } diff --git a/vendors/groc/groq.go b/vendors/groc/groq.go new file mode 100644 index 0000000..bfb18ad --- /dev/null +++ b/vendors/groc/groq.go @@ -0,0 +1,15 @@ +package groc + +import ( + "github.com/danielmiessler/fabric/vendors/openai" +) + +func NewClient() (ret *Client) { + ret = &Client{} + ret.Client = openai.NewClientCompatible("Groq", "https://api.groq.com/openai/v1", nil) + return +} + +type Client struct { + *openai.Client +} diff --git a/vendors/grocq/grocq.go b/vendors/grocq/grocq.go deleted file mode 100644 index d96fc98..0000000 --- a/vendors/grocq/grocq.go +++ /dev/null @@ -1,23 +0,0 @@ -package grocq - -import ( - "github.com/danielmiessler/fabric/vendors/openai" - goopenai "github.com/sashabaranov/go-openai" -) - -func NewClient() (ret *Client) { - ret = &Client{} - ret.Client = openai.NewClientCompatible("Grocq", ret.configure) - return -} - -type Client struct { - *openai.Client -} - -func (oi *Client) configure() (err error) { - config := goopenai.DefaultConfig(oi.ApiKey.Value) - config.BaseURL = "https://api.groq.com/openai/v1" - oi.ApiClient = goopenai.NewClientWithConfig(config) - return -} diff --git a/vendors/ollama/ollama.go b/vendors/ollama/ollama.go index 4cca36a..a10bb0d 100644 --- a/vendors/ollama/ollama.go +++ b/vendors/ollama/ollama.go @@ -43,7 +43,7 @@ func (o *Client) configure() (err error) { return } - o.client = ollamaapi.NewClient(o.apiUrl, &http.Client{Timeout: 10000 * time.Millisecond}) + o.client = ollamaapi.NewClient(o.apiUrl, &http.Client{Timeout: 1200000 * time.Millisecond}) return } @@ -86,8 +86,6 @@ func (o *Client) Send(msgs []*common.Message, opts *common.ChatOptions) (ret str req.Stream = &bf respFunc := func(resp ollamaapi.ChatResponse) (streamErr error) { - fmt.Print(resp.Message.Content) - fmt.Printf("FRED ==> \n") ret = resp.Message.Content return } diff --git a/vendors/openai/openai.go b/vendors/openai/openai.go index 9333cfc..9074378 100644 --- a/vendors/openai/openai.go +++ b/vendors/openai/openai.go @@ -13,10 +13,10 @@ import ( ) func NewClient() (ret *Client) { - return NewClientCompatible("OpenAI", nil) + return NewClientCompatible("OpenAI", "https://api.openai.com/v1", nil) } -func NewClientCompatible(vendorName string, configureCustom func() error) (ret *Client) { +func NewClientCompatible(vendorName string, defaultBaseUrl string, configureCustom func() error) (ret *Client) { ret = &Client{} if configureCustom == nil { @@ -29,19 +29,26 @@ func NewClientCompatible(vendorName string, configureCustom func() error) (ret * ConfigureCustom: configureCustom, } - ret.ApiKey = ret.AddSetupQuestion("API key", true) + ret.ApiKey = ret.AddSetupQuestion("API Key", true) + ret.ApiBaseURL = ret.AddSetupQuestion("API Base URL", false) + ret.ApiBaseURL.Value = defaultBaseUrl return } type Client struct { *common.Configurable - ApiKey *common.SetupQuestion - ApiClient *openai.Client + ApiKey *common.SetupQuestion + ApiBaseURL *common.SetupQuestion + ApiClient *openai.Client } func (o *Client) configure() (ret error) { - o.ApiClient = openai.NewClient(o.ApiKey.Value) + config := openai.DefaultConfig(o.ApiKey.Value) + if o.ApiBaseURL.Value != "" { + config.BaseURL = o.ApiBaseURL.Value + } + o.ApiClient = openai.NewClientWithConfig(config) return } diff --git a/vendors/vendor.go b/vendors/vendor.go new file mode 100644 index 0000000..bf01aaf --- /dev/null +++ b/vendors/vendor.go @@ -0,0 +1,17 @@ +package vendors + +import ( + "bytes" + "github.com/danielmiessler/fabric/common" +) + +type Vendor interface { + GetName() string + IsConfigured() bool + Configure() error + ListModels() ([]string, error) + SendStream([]*common.Message, *common.ChatOptions, chan string) error + Send([]*common.Message, *common.ChatOptions) (string, error) + Setup() error + SetupFillEnvFileContent(*bytes.Buffer) +} diff --git a/youtube/youtube.go b/youtube/youtube.go index c0fbc55..0e935a1 100644 --- a/youtube/youtube.go +++ b/youtube/youtube.go @@ -1,7 +1,18 @@ package youtube import ( + "context" + "encoding/json" + "flag" + "fmt" + "github.com/anaskhan96/soup" "github.com/danielmiessler/fabric/common" + "google.golang.org/api/option" + "google.golang.org/api/youtube/v3" + "log" + "regexp" + "strconv" + "strings" ) func NewYouTube() (ret *YouTube) { @@ -22,4 +33,218 @@ func NewYouTube() (ret *YouTube) { type YouTube struct { *common.Configurable ApiKey *common.SetupQuestion + + service *youtube.Service +} + +func (o *YouTube) initService() (err error) { + if o.service == nil { + ctx := context.Background() + o.service, err = youtube.NewService(ctx, option.WithAPIKey(o.ApiKey.Value)) + } + return +} + +func (o *YouTube) GetVideoId(url string) (ret string, err error) { + if err = o.initService(); err != nil { + return + } + + pattern := `(?:https?:\/\/)?(?:www\.)?(?:youtube\.com\/(?:[^\/\n\s]+\/\S+\/|(?:v|e(?:mbed)?)\/|\S*?[?&]v=)|youtu\.be\/)([a-zA-Z0-9_-]{11})` + re := regexp.MustCompile(pattern) + match := re.FindStringSubmatch(url) + if len(match) > 1 { + ret = match[1] + } else { + err = fmt.Errorf("invalid YouTube URL, can't get video ID") + } + return +} + +func (o *YouTube) GrabTranscriptForUrl(url string) (ret string, err error) { + var videoId string + if videoId, err = o.GetVideoId(url); err != nil { + return + } + return o.GrabTranscript(videoId) +} + +func (o *YouTube) GrabTranscript(videoId string) (ret string, err error) { + var transcript string + if transcript, err = o.GrabTranscriptBase(videoId); err != nil { + err = fmt.Errorf("transcript not available. (%v)", err) + return + } + + // Parse the XML transcript + doc := soup.HTMLParse(transcript) + // Extract the text content from the tags + textTags := doc.FindAll("text") + var textBuilder strings.Builder + for _, textTag := range textTags { + textBuilder.WriteString(textTag.Text()) + textBuilder.WriteString(" ") + ret = textBuilder.String() + } + return +} + +func (o *YouTube) GrabTranscriptBase(videoId string) (ret string, err error) { + if err = o.initService(); err != nil { + return + } + + url := "https://www.youtube.com/watch?v=" + videoId + var resp string + if resp, err = soup.Get(url); err != nil { + return + } + + doc := soup.HTMLParse(resp) + scriptTags := doc.FindAll("script") + for _, scriptTag := range scriptTags { + if strings.Contains(scriptTag.Text(), "captionTracks") { + regex := regexp.MustCompile(`"captionTracks":(\[.*?\])`) + match := regex.FindStringSubmatch(scriptTag.Text()) + if len(match) > 1 { + var captionTracks []struct { + BaseURL string `json:"baseUrl"` + } + + if err = json.Unmarshal([]byte(match[1]), &captionTracks); err != nil { + return + } + + if len(captionTracks) > 0 { + transcriptURL := captionTracks[0].BaseURL + ret, err = soup.Get(transcriptURL) + return + } + } + } + } + err = fmt.Errorf("transcript not found") + return +} + +func (o *YouTube) GrabComments(videoId string) (ret []string, err error) { + if err = o.initService(); err != nil { + return + } + + call := o.service.CommentThreads.List([]string{"snippet", "replies"}).VideoId(videoId).TextFormat("plainText").MaxResults(100) + var response *youtube.CommentThreadListResponse + if response, err = call.Do(); err != nil { + log.Printf("Failed to fetch comments: %v", err) + return + } + + for _, item := range response.Items { + topLevelComment := item.Snippet.TopLevelComment.Snippet.TextDisplay + ret = append(ret, topLevelComment) + + if item.Replies != nil { + for _, reply := range item.Replies.Comments { + replyText := reply.Snippet.TextDisplay + ret = append(ret, " - "+replyText) + } + } + } + return +} + +func (o *YouTube) GrabDurationForUrl(url string) (ret int, err error) { + if err = o.initService(); err != nil { + return + } + + var videoId string + if videoId, err = o.GetVideoId(url); err != nil { + return + } + return o.GrabDuration(videoId) +} + +func (o *YouTube) GrabDuration(videoId string) (ret int, err error) { + var videoResponse *youtube.VideoListResponse + if videoResponse, err = o.service.Videos.List([]string{"contentDetails"}).Id(videoId).Do(); err != nil { + err = fmt.Errorf("error getting video details: %v", err) + return + } + + durationStr := videoResponse.Items[0].ContentDetails.Duration + + matches := regexp.MustCompile(`(?i)PT(?:(\d+)H)?(?:(\d+)M)?(?:(\d+)S)?`).FindStringSubmatch(durationStr) + if len(matches) == 0 { + return 0, fmt.Errorf("invalid duration string: %s", durationStr) + } + + hours, _ := strconv.Atoi(matches[1]) + minutes, _ := strconv.Atoi(matches[2]) + seconds, _ := strconv.Atoi(matches[3]) + + ret = hours*60 + minutes + seconds/60 + + return +} + +func (o *YouTube) Grab(url string, options *Options) (ret *VideoInfo, err error) { + var videoId string + if videoId, err = o.GetVideoId(url); err != nil { + return + } + + ret = &VideoInfo{} + + if options.Duration { + if ret.Duration, err = o.GrabDuration(videoId); err != nil { + err = fmt.Errorf("error parsing video duration: %v", err) + return + } + + } + + if options.Comments { + if ret.Comments, err = o.GrabComments(videoId); err != nil { + err = fmt.Errorf("error getting comments: %v", err) + return + } + } + + if options.Transcript { + if ret.Transcript, err = o.GrabTranscript(videoId); err != nil { + return + } + } + return +} + +type Options struct { + Duration bool + Transcript bool + Comments bool + Lang string +} + +type VideoInfo struct { + Transcript string `json:"transcript"` + Duration int `json:"duration"` + Comments []string `json:"comments"` +} + +func (o *YouTube) GrabByFlags() (ret *VideoInfo, err error) { + options := &Options{} + flag.BoolVar(&options.Duration, "duration", false, "Output only the duration") + flag.BoolVar(&options.Transcript, "transcript", false, "Output only the transcript") + flag.BoolVar(&options.Comments, "comments", false, "Output the comments on the video") + flag.StringVar(&options.Lang, "lang", "en", "Language for the transcript (default: English)") + flag.Parse() + + if flag.NArg() == 0 { + log.Fatal("Error: No URL provided.") + } + + url := flag.Arg(0) + ret, err = o.Grab(url, options) + return }