m3tam3re/fabric
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'main' of github.com:danielmiessler/fabric

Browse Source
This commit is contained in:
Daniel Miessler 2024-10-01 12:24:34 -07:00
commit f54a052533
16 changed files with 318 additions and 62 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.github/workflows/release.yml vendored
View File

@ -64,7 +64,7 @@ jobs:
GOOS: ${{ env.OS }}
GOARCH: ${{ matrix.arch }}
run: |
go build -o fabric-${OS}-${{ matrix.arch }} .
go build -ldflags "-X main.version=$(git describe --tags --abbrev=0)" -o fabric-${OS}-${{ matrix.arch }} .
- name: Build binary on Windows
if: matrix.os == 'windows-latest'
@ -72,7 +72,7 @@ jobs:
GOOS: windows
GOARCH: ${{ matrix.arch }}
run: |
go build -o fabric-windows-${{ matrix.arch }}.exe .
go build -ldflags "-X main.version=$(git describe --tags --abbrev=0)" -o fabric-windows-${{ matrix.arch }}.exe .
- name: Upload build artifact
if: matrix.os != 'windows-latest'

59
.github/workflows/update-version.yml vendored Normal file
View File

@ -0,0 +1,59 @@
name: Update Version File
on:
push:
branches:
- main # Or whichever branch you want to monitor
tags:
- '*' # Trigger on any new tag
permissions:
contents: write # Ensure the workflow has write permissions
jobs:
update-version:
runs-on: ubuntu-latest
steps:
- name: Checkout repository
uses: actions/checkout@v4
with:
fetch-depth: 0 # Fetch all history to include tags
- name: Set up Git
run: |
git config user.name "github-actions[bot]"
git config user.email "github-actions[bot]@users.noreply.github.com"
- name: Get the latest tag
id: get_latest_tag
run: |
latest_tag=$(git describe --tags --abbrev=0)
echo "Latest tag is: $latest_tag"
echo "::set-output name=tag::$latest_tag"
- name: Get the latest commit hash
id: get_commit_hash
run: |
commit_hash=$(git rev-parse --short HEAD)
echo "Commit hash is: $commit_hash"
echo "::set-output name=commit_hash::$commit_hash"
- name: Update version.go file
run: |
latest_tag=${{ steps.get_latest_tag.outputs.tag }}
commit_hash=${{ steps.get_commit_hash.outputs.commit_hash }}
echo "package main" > version.go
echo "" >> version.go
echo "var version = \"${latest_tag}-${commit_hash}\"" >> version.go
- name: Commit changes
run: |
git add version.go
git commit -m "Update version to ${{ steps.get_latest_tag.outputs.tag }} and commit ${{ steps.get_commit_hash.outputs.commit_hash }}"
- name: Push changes
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Use GITHUB_TOKEN to authenticate the push
run: |
git push origin main # Or the relevant branch

59
README.md
View File

@ -113,6 +113,29 @@ Fabric has Patterns for all sorts of life and work activities, including:
## Installation
To install Fabric, you can use the latest release binaries or install it from the source.
### Get Latest Release Binaries
```bash
# Windows:
curl -L https://github.com/danielmiessler/fabric/releases/latest/download/fabric-windows-amd64.exe > fabric.exe && fabric.exe --version
# MacOS (arm64):
curl -L https://github.com/danielmiessler/fabric/releases/latest/download/fabric-darwin-arm64 > fabric && chmod +x fabric && ./fabric --version
# MacOS (amd64):
curl -L https://github.com/danielmiessler/fabric/releases/latest/download/fabric-darwin-amd64 > fabric && chmod +x fabric && ./fabric --version
# Linux (amd64):
curl -L https://github.com/danielmiessler/fabric/releases/latest/download/fabric-linux-amd64 > fabric && chmod +x fabric && ./fabric --version
# Linux (arm64):
curl -L https://github.com/danielmiessler/fabric/releases/latest/download/fabric-linux-arm64 > fabric && chmod +x fabric && ./fabric --version
```
### From Source
To install Fabric, [make sure Go is installed](https://go.dev/doc/install), and then run the following command.
```bash
@ -173,7 +196,7 @@ Then [set your environmental variables](#environmental-variables) as shown above
The great thing about Go is that it's super easy to upgrade. Just run the same command you used to install it in the first place and you'll always get the latest version.
```bash
go install github.com/danielmiessler/fabric@latest
go install -ldflags "-X main.version=$(git describe --tags --always)" github.com/danielmiessler/fabric@latest
```
## Usage
@ -211,13 +234,17 @@ Application Options:
-o, --output= Output to file
-n, --latest= Number of latest patterns to list (default: 0)
-d, --changeDefaultModel Change default model
-y, --youtube= YouTube video url to grab transcript, comments from it and send to chat
--transcript Grab transcript from YouTube video and send to chat
-y, --youtube= YouTube video "URL" to grab transcript, comments from it and send to chat
--transcript Grab transcript from YouTube video and send to chat (it used per default).
--comments Grab comments from YouTube video and send to chat
--dry-run Show what would be sent to the model without actually sending it
-g, --language= Specify the Language Code for the chat, e.g. -g=en -g=zh
-u, --scrape_url= Scrape website URL to markdown using Jina AI
-q, --scrape_question= Search question using Jina AI
-e, --seed= Seed to be used for LMM generation
-w, --wipecontext= Wipe context
-W, --wipesession= Wipe session
--dry-run Show what would be sent to the model without actually sending it
--version Print current version
Help Options:
-h, --help Show this help message
@ -261,7 +288,7 @@ pbpaste | fabric --stream --pattern analyze_claims
3. Run the `extract_wisdom` Pattern with the `--stream` option to get immediate and streaming results from any Youtube video (much like in the original introduction video).
```bash
yt --transcript https://youtube.com/watch?v=uXs-zPc63kM | fabric --stream --pattern extract_wisdom
fabric -y "https://youtube.com/watch?v=uXs-zPc63kM" | --stream --pattern extract_wisdom
```
4. Create patterns- you must create a .md file with the pattern and save it to ~/.config/fabric/patterns/[yourpatternname].
@ -302,26 +329,6 @@ This feature works with all openai and ollama models but does NOT work with clau
Fabric also makes use of some core helper apps (tools) to make it easier to integrate with your various workflows. Here are some examples:
`yt` is a helper command that extracts the transcript from a YouTube video. You can use it like this:
```bash
yt https://www.youtube.com/watch?v=lQVcbY52_gY
```
This will return the transcript from the video, which you can then pipe into Fabric like this:
```bash
yt https://www.youtube.com/watch?v=lQVcbY52_gY | fabric --pattern extract_wisdom
```
### `yt` Installation
To install `yt`, install it the same way as you install Fabric, just with a different repo name.
```bash
go install github.com/danielmiessler/yt@latest
```
Be sure to add your `YOUTUBE_API_KEY` to `~/.config/fabric/.env`.
### `to_pdf`
`to_pdf` is a helper command that converts LaTeX files to PDF format. You can use it like this:
@ -345,7 +352,7 @@ This will create a PDF file named `output.pdf` in the current directory.
To install `to_pdf`, install it the same way as you install Fabric, just with a different repo name.
```bash
go install github.com/danielmiessler/fabric/to_pdf/to_pdf@latest
go install github.com/danielmiessler/fabric/to_pdf@latest
```
Make sure you have a LaTeX distribution (like TeX Live or MiKTeX) installed on your system, as `to_pdf` requires `pdflatex` to be available in your system's PATH.

37
cli/cli.go
View File

@ -12,11 +12,14 @@ import (
)
// Cli Controls the cli. It takes in the flags and runs the appropriate functions
func Cli() (message string, err error) {
func Cli(version string) (message string, err error) {
var currentFlags *Flags
if currentFlags, err = Init(); err != nil {
// we need to reset error, because we don't want to show double help messages
err = nil
return
}
if currentFlags.Version {
fmt.Println(version)
return
}
@ -95,6 +98,18 @@ func Cli() (message string, err error) {
return
}
// if the wipe context flag is set, run the wipe context function
if currentFlags.WipeContext != "" {
err = fabricDb.Contexts.Delete(currentFlags.WipeContext)
return
}
// if the wipe session flag is set, run the wipe session function
if currentFlags.WipeSession != "" {
err = fabricDb.Sessions.Delete(currentFlags.WipeSession)
return
}
// if the interactive flag is set, run the interactive function
// if currentFlags.Interactive {
// interactive.Interactive()
@ -115,11 +130,15 @@ func Cli() (message string, err error) {
if !currentFlags.YouTubeComments || currentFlags.YouTubeTranscript {
var transcript string
if transcript, err = fabric.YouTube.GrabTranscript(videoId); err != nil {
var language = "en"
if currentFlags.Language != "" {
language = currentFlags.Language
}
if transcript, err = fabric.YouTube.GrabTranscript(videoId, language); err != nil {
return
}
fmt.Println(transcript)
// fmt.Println(transcript)
currentFlags.AppendMessage(transcript)
}
@ -132,13 +151,14 @@ func Cli() (message string, err error) {
commentsString := strings.Join(comments, "\n")
fmt.Println(commentsString)
// fmt.Println(commentsString)
currentFlags.AppendMessage(commentsString)
}
if currentFlags.Pattern == "" {
// if the pattern flag is not set, we wanted only to grab the transcript or comments
fmt.Println(currentFlags.Message)
return
}
}
@ -150,7 +170,7 @@ func Cli() (message string, err error) {
return
}
fmt.Println(message)
//fmt.Println(message)
currentFlags.AppendMessage(message)
}
@ -161,13 +181,14 @@ func Cli() (message string, err error) {
return
}
fmt.Println(message)
//fmt.Println(message)
currentFlags.AppendMessage(message)
}
if currentFlags.Pattern == "" {
// if the pattern flag is not set, we wanted only to grab the url or get the answer to the question
fmt.Println(currentFlags.Message)
return
}
}

11
cli/cli_test.go
View File

@ -1,6 +1,7 @@
package cli
import (
"github.com/danielmiessler/fabric/core"
"os"
"testing"
@ -9,8 +10,14 @@ import (
)
func TestCli(t *testing.T) {
message, err := Cli()
assert.NoError(t, err)
t.Skip("Skipping test for now, collision with flag -t")
originalArgs := os.Args
defer func() { os.Args = originalArgs }()
os.Args = []string{os.Args[0]}
message, err := Cli("test")
assert.Error(t, err)
assert.Equal(t, core.NoSessionPatternUserMessages, err.Error())
assert.Empty(t, message)
}

11
cli/flags.go
View File

@ -37,13 +37,17 @@ type Flags struct {
Output string `short:"o" long:"output" description:"Output to file" default:""`
LatestPatterns string `short:"n" long:"latest" description:"Number of latest patterns to list" default:"0"`
ChangeDefaultModel bool `short:"d" long:"changeDefaultModel" description:"Change default model"`
YouTube string `short:"y" long:"youtube" description:"YouTube video url to grab transcript, comments from it and send to chat"`
YouTubeTranscript bool `long:"transcript" description:"Grab transcript from YouTube video and send to chat"`
YouTube string `short:"y" long:"youtube" description:"YouTube video \"URL\" to grab transcript, comments from it and send to chat"`
YouTubeTranscript bool `long:"transcript" description:"Grab transcript from YouTube video and send to chat (it used per default)."`
YouTubeComments bool `long:"comments" description:"Grab comments from YouTube video and send to chat"`
DryRun bool `long:"dry-run" description:"Show what would be sent to the model without actually sending it"`
Language string `short:"g" long:"language" description:"Specify the Language Code for the chat, e.g. -g=en -g=zh" default:""`
ScrapeURL string `short:"u" long:"scrape_url" description:"Scrape website URL to markdown using Jina AI"`
ScrapeQuestion string `short:"q" long:"scrape_question" description:"Search question using Jina AI"`
Seed int `short:"e" long:"seed" description:"Seed to be used for LMM generation"`
WipeContext string `short:"w" long:"wipecontext" description:"Wipe context"`
WipeSession string `short:"W" long:"wipesession" description:"Wipe session"`
DryRun bool `long:"dry-run" description:"Show what would be sent to the model without actually sending it"`
Version bool `long:"version" description:"Print current version"`
}
// Init Initialize flags. returns a Flags struct and an error
@ -99,6 +103,7 @@ func (o *Flags) BuildChatOptions() (ret *common.ChatOptions) {
PresencePenalty: o.PresencePenalty,
FrequencyPenalty: o.FrequencyPenalty,
Raw: o.Raw,
Seed: o.Seed,
}
return
}

22
cli/flags_test.go
View File

@ -53,6 +53,7 @@ func TestBuildChatOptions(t *testing.T) {
TopP: 0.9,
PresencePenalty: 0.1,
FrequencyPenalty: 0.2,
Seed: 1,
}
expectedOptions := &common.ChatOptions{
@ -61,6 +62,27 @@ func TestBuildChatOptions(t *testing.T) {
PresencePenalty: 0.1,
FrequencyPenalty: 0.2,
Raw: false,
Seed: 1,
}
options := flags.BuildChatOptions()
assert.Equal(t, expectedOptions, options)
}
func TestBuildChatOptionsDefaultSeed(t *testing.T) {
flags := &Flags{
Temperature: 0.8,
TopP: 0.9,
PresencePenalty: 0.1,
FrequencyPenalty: 0.2,
}
expectedOptions := &common.ChatOptions{
Temperature: 0.8,
TopP: 0.9,
PresencePenalty: 0.1,
FrequencyPenalty: 0.2,
Raw: false,
Seed: 0,
}
options := flags.BuildChatOptions()
assert.Equal(t, expectedOptions, options)

5
common/configurable.go
View File

@ -103,8 +103,9 @@ func (o *Setting) IsDefined() bool {
}
func (o *Setting) Configure() error {
if o.Value == "" {
o.Value = os.Getenv(o.EnvVariable)
envValue := os.Getenv(o.EnvVariable)
if envValue != "" {
o.Value = envValue
}
return o.IsValidErr()
}

1
common/domain.go
View File

@ -23,6 +23,7 @@ type ChatOptions struct {
PresencePenalty float64
FrequencyPenalty float64
Raw bool
Seed int
}
// NormalizeMessages remove empty messages and ensure messages order user-assist-user

4
core/fabric.go
View File

@ -30,6 +30,8 @@ import (
const DefaultPatternsGitRepoUrl = "https://github.com/danielmiessler/fabric.git"
const DefaultPatternsGitRepoFolder = "patterns"
const NoSessionPatternUserMessages = "no session, pattern or user messages provided"
func NewFabric(db *db.Db) (ret *Fabric, err error) {
ret = NewFabricBase(db)
err = ret.Configure()
@ -279,7 +281,7 @@ func (o *Chat) BuildChatSession(raw bool) (ret *db.Session, err error) {
if ret.IsEmpty() {
ret = nil
err = fmt.Errorf("no session, pattern or user messages provided")
err = fmt.Errorf(NoSessionPatternUserMessages)
}
return
}

2
go.mod
View File

@ -16,6 +16,7 @@ require (
github.com/samber/lo v1.47.0
github.com/sashabaranov/go-openai v1.30.0
github.com/stretchr/testify v1.9.0
golang.org/x/text v0.18.0
google.golang.org/api v0.197.0
)
@ -61,7 +62,6 @@ require (
golang.org/x/oauth2 v0.23.0 // indirect
golang.org/x/sync v0.8.0 // indirect
golang.org/x/sys v0.25.0 // indirect
golang.org/x/text v0.18.0 // indirect
golang.org/x/time v0.6.0 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20240903143218-8af14fe29dc1 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect

5
main.go
View File

@ -2,14 +2,15 @@ package main
import (
"fmt"
"github.com/jessevdk/go-flags"
"os"
"github.com/danielmiessler/fabric/cli"
)
func main() {
_, err := cli.Cli()
if err != nil {
_, err := cli.Cli(version)
if err != nil && !flags.WroteHelp(err) {
fmt.Printf("%s\n", err)
os.Exit(1)
}

14
vendors/openai/openai.go vendored
View File

@ -5,6 +5,7 @@ import (
"errors"
"fmt"
"io"
"log/slog"
"github.com/danielmiessler/fabric/common"
"github.com/samber/lo"
@ -111,6 +112,7 @@ func (o *Client) Send(ctx context.Context, msgs []*common.Message, opts *common.
}
if len(resp.Choices) > 0 {
ret = resp.Choices[0].Message.Content
slog.Debug("SystemFingerprint: " + resp.SystemFingerprint)
}
return
}
@ -128,6 +130,7 @@ func (o *Client) buildChatCompletionRequest(
Messages: messages,
}
} else {
if opts.Seed == 0 {
ret = goopenai.ChatCompletionRequest{
Model: opts.Model,
Temperature: float32(opts.Temperature),
@ -136,6 +139,17 @@ func (o *Client) buildChatCompletionRequest(
FrequencyPenalty: float32(opts.FrequencyPenalty),
Messages: messages,
}
} else {
ret = goopenai.ChatCompletionRequest{
Model: opts.Model,
Temperature: float32(opts.Temperature),
TopP: float32(opts.TopP),
PresencePenalty: float32(opts.PresencePenalty),
FrequencyPenalty: float32(opts.FrequencyPenalty),
Messages: messages,
Seed: &opts.Seed,
}
}
}
return
}

102
vendors/openai/openai_test.go vendored Normal file
View File

@ -0,0 +1,102 @@
package openai
import (
"testing"
"github.com/danielmiessler/fabric/common"
"github.com/sashabaranov/go-openai"
goopenai "github.com/sashabaranov/go-openai"
"github.com/stretchr/testify/assert"
)
func TestBuildChatCompletionRequestPinSeed(t *testing.T) {
var msgs []*common.Message
for i := 0; i < 2; i++ {
msgs = append(msgs, &common.Message{
Role: "User",
Content: "My msg",
})
}
opts := &common.ChatOptions{
Temperature: 0.8,
TopP: 0.9,
PresencePenalty: 0.1,
FrequencyPenalty: 0.2,
Raw: false,
Seed: 1,
}
var expectedMessages []openai.ChatCompletionMessage
for i := 0; i < 2; i++ {
expectedMessages = append(expectedMessages,
openai.ChatCompletionMessage{
Role: msgs[i].Role,
Content: msgs[i].Content,
},
)
}
var expectedRequest = goopenai.ChatCompletionRequest{
Model: opts.Model,
Temperature: float32(opts.Temperature),
TopP: float32(opts.TopP),
PresencePenalty: float32(opts.PresencePenalty),
FrequencyPenalty: float32(opts.FrequencyPenalty),
Messages: expectedMessages,
Seed: &opts.Seed,
}
var client = NewClient()
request := client.buildChatCompletionRequest(msgs, opts)
assert.Equal(t, expectedRequest, request)
}
func TestBuildChatCompletionRequestNilSeed(t *testing.T) {
var msgs []*common.Message
for i := 0; i < 2; i++ {
msgs = append(msgs, &common.Message{
Role: "User",
Content: "My msg",
})
}
opts := &common.ChatOptions{
Temperature: 0.8,
TopP: 0.9,
PresencePenalty: 0.1,
FrequencyPenalty: 0.2,
Raw: false,
Seed: 0,
}
var expectedMessages []openai.ChatCompletionMessage
for i := 0; i < 2; i++ {
expectedMessages = append(expectedMessages,
openai.ChatCompletionMessage{
Role: msgs[i].Role,
Content: msgs[i].Content,
},
)
}
var expectedRequest = goopenai.ChatCompletionRequest{
Model: opts.Model,
Temperature: float32(opts.Temperature),
TopP: float32(opts.TopP),
PresencePenalty: float32(opts.PresencePenalty),
FrequencyPenalty: float32(opts.FrequencyPenalty),
Messages: expectedMessages,
Seed: nil,
}
var client = NewClient()
request := client.buildChatCompletionRequest(msgs, opts)
assert.Equal(t, expectedRequest, request)
}

3
version.go Normal file
View File

@ -0,0 +1,3 @@
package main
var version = "v1.4.31-5d3e0da"

27
youtube/youtube.go
View File

@ -10,6 +10,7 @@ import (
"google.golang.org/api/option"
"google.golang.org/api/youtube/v3"
"log"
"net/url"
"regexp"
"strconv"
"strings"
@ -61,17 +62,17 @@ func (o *YouTube) GetVideoId(url string) (ret string, err error) {
return
}
func (o *YouTube) GrabTranscriptForUrl(url string) (ret string, err error) {
func (o *YouTube) GrabTranscriptForUrl(url string, language string) (ret string, err error) {
var videoId string
if videoId, err = o.GetVideoId(url); err != nil {
return
}
return o.GrabTranscript(videoId)
return o.GrabTranscript(videoId, language)
}
func (o *YouTube) GrabTranscript(videoId string) (ret string, err error) {
func (o *YouTube) GrabTranscript(videoId string, language string) (ret string, err error) {
var transcript string
if transcript, err = o.GrabTranscriptBase(videoId); err != nil {
if transcript, err = o.GrabTranscriptBase(videoId, language); err != nil {
err = fmt.Errorf("transcript not available. (%v)", err)
return
}
@ -89,14 +90,14 @@ func (o *YouTube) GrabTranscript(videoId string) (ret string, err error) {
return
}
func (o *YouTube) GrabTranscriptBase(videoId string) (ret string, err error) {
func (o *YouTube) GrabTranscriptBase(videoId string, language string) (ret string, err error) {
if err = o.initService(); err != nil {
return
}
url := "https://www.youtube.com/watch?v=" + videoId
watchUrl := "https://www.youtube.com/watch?v=" + videoId
var resp string
if resp, err = soup.Get(url); err != nil {
if resp, err = soup.Get(watchUrl); err != nil {
return
}
@ -117,6 +118,16 @@ func (o *YouTube) GrabTranscriptBase(videoId string) (ret string, err error) {
if len(captionTracks) > 0 {
transcriptURL := captionTracks[0].BaseURL
for _, captionTrack := range captionTracks {
parsedUrl, error := url.Parse(captionTrack.BaseURL)
if error != nil {
err = fmt.Errorf("error parsing caption track")
}
parsedUrlParams, _ := url.ParseQuery(parsedUrl.RawQuery)
if parsedUrlParams["lang"][0] == language {
transcriptURL = captionTrack.BaseURL
}
}
ret, err = soup.Get(transcriptURL)
return
}
@ -212,7 +223,7 @@ func (o *YouTube) Grab(url string, options *Options) (ret *VideoInfo, err error)
}
if options.Transcript {
if ret.Transcript, err = o.GrabTranscript(videoId); err != nil {
if ret.Transcript, err = o.GrabTranscript(videoId, "en"); err != nil {
return
}
}