Merge branch 'main' of github.com:danielmiessler/fabric
This commit is contained in:
commit
f54a052533
4
.github/workflows/release.yml
vendored
4
.github/workflows/release.yml
vendored
@ -64,7 +64,7 @@ jobs:
|
||||
GOOS: ${{ env.OS }}
|
||||
GOARCH: ${{ matrix.arch }}
|
||||
run: |
|
||||
go build -o fabric-${OS}-${{ matrix.arch }} .
|
||||
go build -ldflags "-X main.version=$(git describe --tags --abbrev=0)" -o fabric-${OS}-${{ matrix.arch }} .
|
||||
|
||||
- name: Build binary on Windows
|
||||
if: matrix.os == 'windows-latest'
|
||||
@ -72,7 +72,7 @@ jobs:
|
||||
GOOS: windows
|
||||
GOARCH: ${{ matrix.arch }}
|
||||
run: |
|
||||
go build -o fabric-windows-${{ matrix.arch }}.exe .
|
||||
go build -ldflags "-X main.version=$(git describe --tags --abbrev=0)" -o fabric-windows-${{ matrix.arch }}.exe .
|
||||
|
||||
- name: Upload build artifact
|
||||
if: matrix.os != 'windows-latest'
|
||||
|
59
.github/workflows/update-version.yml
vendored
Normal file
59
.github/workflows/update-version.yml
vendored
Normal file
@ -0,0 +1,59 @@
|
||||
name: Update Version File
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- main # Or whichever branch you want to monitor
|
||||
tags:
|
||||
- '*' # Trigger on any new tag
|
||||
|
||||
permissions:
|
||||
contents: write # Ensure the workflow has write permissions
|
||||
|
||||
jobs:
|
||||
update-version:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0 # Fetch all history to include tags
|
||||
|
||||
- name: Set up Git
|
||||
run: |
|
||||
git config user.name "github-actions[bot]"
|
||||
git config user.email "github-actions[bot]@users.noreply.github.com"
|
||||
|
||||
- name: Get the latest tag
|
||||
id: get_latest_tag
|
||||
run: |
|
||||
latest_tag=$(git describe --tags --abbrev=0)
|
||||
echo "Latest tag is: $latest_tag"
|
||||
echo "::set-output name=tag::$latest_tag"
|
||||
|
||||
- name: Get the latest commit hash
|
||||
id: get_commit_hash
|
||||
run: |
|
||||
commit_hash=$(git rev-parse --short HEAD)
|
||||
echo "Commit hash is: $commit_hash"
|
||||
echo "::set-output name=commit_hash::$commit_hash"
|
||||
|
||||
- name: Update version.go file
|
||||
run: |
|
||||
latest_tag=${{ steps.get_latest_tag.outputs.tag }}
|
||||
commit_hash=${{ steps.get_commit_hash.outputs.commit_hash }}
|
||||
echo "package main" > version.go
|
||||
echo "" >> version.go
|
||||
echo "var version = \"${latest_tag}-${commit_hash}\"" >> version.go
|
||||
|
||||
- name: Commit changes
|
||||
run: |
|
||||
git add version.go
|
||||
git commit -m "Update version to ${{ steps.get_latest_tag.outputs.tag }} and commit ${{ steps.get_commit_hash.outputs.commit_hash }}"
|
||||
|
||||
- name: Push changes
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Use GITHUB_TOKEN to authenticate the push
|
||||
run: |
|
||||
git push origin main # Or the relevant branch
|
59
README.md
59
README.md
@ -113,6 +113,29 @@ Fabric has Patterns for all sorts of life and work activities, including:
|
||||
|
||||
## Installation
|
||||
|
||||
To install Fabric, you can use the latest release binaries or install it from the source.
|
||||
|
||||
### Get Latest Release Binaries
|
||||
|
||||
```bash
|
||||
# Windows:
|
||||
curl -L https://github.com/danielmiessler/fabric/releases/latest/download/fabric-windows-amd64.exe > fabric.exe && fabric.exe --version
|
||||
|
||||
# MacOS (arm64):
|
||||
curl -L https://github.com/danielmiessler/fabric/releases/latest/download/fabric-darwin-arm64 > fabric && chmod +x fabric && ./fabric --version
|
||||
|
||||
# MacOS (amd64):
|
||||
curl -L https://github.com/danielmiessler/fabric/releases/latest/download/fabric-darwin-amd64 > fabric && chmod +x fabric && ./fabric --version
|
||||
|
||||
# Linux (amd64):
|
||||
curl -L https://github.com/danielmiessler/fabric/releases/latest/download/fabric-linux-amd64 > fabric && chmod +x fabric && ./fabric --version
|
||||
|
||||
# Linux (arm64):
|
||||
curl -L https://github.com/danielmiessler/fabric/releases/latest/download/fabric-linux-arm64 > fabric && chmod +x fabric && ./fabric --version
|
||||
```
|
||||
|
||||
### From Source
|
||||
|
||||
To install Fabric, [make sure Go is installed](https://go.dev/doc/install), and then run the following command.
|
||||
|
||||
```bash
|
||||
@ -173,7 +196,7 @@ Then [set your environmental variables](#environmental-variables) as shown above
|
||||
|
||||
The great thing about Go is that it's super easy to upgrade. Just run the same command you used to install it in the first place and you'll always get the latest version.
|
||||
```bash
|
||||
go install github.com/danielmiessler/fabric@latest
|
||||
go install -ldflags "-X main.version=$(git describe --tags --always)" github.com/danielmiessler/fabric@latest
|
||||
```
|
||||
|
||||
## Usage
|
||||
@ -211,13 +234,17 @@ Application Options:
|
||||
-o, --output= Output to file
|
||||
-n, --latest= Number of latest patterns to list (default: 0)
|
||||
-d, --changeDefaultModel Change default model
|
||||
-y, --youtube= YouTube video url to grab transcript, comments from it and send to chat
|
||||
--transcript Grab transcript from YouTube video and send to chat
|
||||
-y, --youtube= YouTube video "URL" to grab transcript, comments from it and send to chat
|
||||
--transcript Grab transcript from YouTube video and send to chat (it used per default).
|
||||
--comments Grab comments from YouTube video and send to chat
|
||||
--dry-run Show what would be sent to the model without actually sending it
|
||||
-g, --language= Specify the Language Code for the chat, e.g. -g=en -g=zh
|
||||
-u, --scrape_url= Scrape website URL to markdown using Jina AI
|
||||
-q, --scrape_question= Search question using Jina AI
|
||||
-e, --seed= Seed to be used for LMM generation
|
||||
-w, --wipecontext= Wipe context
|
||||
-W, --wipesession= Wipe session
|
||||
--dry-run Show what would be sent to the model without actually sending it
|
||||
--version Print current version
|
||||
|
||||
Help Options:
|
||||
-h, --help Show this help message
|
||||
@ -261,7 +288,7 @@ pbpaste | fabric --stream --pattern analyze_claims
|
||||
3. Run the `extract_wisdom` Pattern with the `--stream` option to get immediate and streaming results from any Youtube video (much like in the original introduction video).
|
||||
|
||||
```bash
|
||||
yt --transcript https://youtube.com/watch?v=uXs-zPc63kM | fabric --stream --pattern extract_wisdom
|
||||
fabric -y "https://youtube.com/watch?v=uXs-zPc63kM" | --stream --pattern extract_wisdom
|
||||
```
|
||||
|
||||
4. Create patterns- you must create a .md file with the pattern and save it to ~/.config/fabric/patterns/[yourpatternname].
|
||||
@ -302,26 +329,6 @@ This feature works with all openai and ollama models but does NOT work with clau
|
||||
|
||||
Fabric also makes use of some core helper apps (tools) to make it easier to integrate with your various workflows. Here are some examples:
|
||||
|
||||
`yt` is a helper command that extracts the transcript from a YouTube video. You can use it like this:
|
||||
```bash
|
||||
yt https://www.youtube.com/watch?v=lQVcbY52_gY
|
||||
```
|
||||
|
||||
This will return the transcript from the video, which you can then pipe into Fabric like this:
|
||||
```bash
|
||||
yt https://www.youtube.com/watch?v=lQVcbY52_gY | fabric --pattern extract_wisdom
|
||||
```
|
||||
|
||||
### `yt` Installation
|
||||
|
||||
To install `yt`, install it the same way as you install Fabric, just with a different repo name.
|
||||
|
||||
```bash
|
||||
go install github.com/danielmiessler/yt@latest
|
||||
```
|
||||
|
||||
Be sure to add your `YOUTUBE_API_KEY` to `~/.config/fabric/.env`.
|
||||
|
||||
### `to_pdf`
|
||||
|
||||
`to_pdf` is a helper command that converts LaTeX files to PDF format. You can use it like this:
|
||||
@ -345,7 +352,7 @@ This will create a PDF file named `output.pdf` in the current directory.
|
||||
To install `to_pdf`, install it the same way as you install Fabric, just with a different repo name.
|
||||
|
||||
```bash
|
||||
go install github.com/danielmiessler/fabric/to_pdf/to_pdf@latest
|
||||
go install github.com/danielmiessler/fabric/to_pdf@latest
|
||||
```
|
||||
|
||||
Make sure you have a LaTeX distribution (like TeX Live or MiKTeX) installed on your system, as `to_pdf` requires `pdflatex` to be available in your system's PATH.
|
||||
|
37
cli/cli.go
37
cli/cli.go
@ -12,11 +12,14 @@ import (
|
||||
)
|
||||
|
||||
// Cli Controls the cli. It takes in the flags and runs the appropriate functions
|
||||
func Cli() (message string, err error) {
|
||||
func Cli(version string) (message string, err error) {
|
||||
var currentFlags *Flags
|
||||
if currentFlags, err = Init(); err != nil {
|
||||
// we need to reset error, because we don't want to show double help messages
|
||||
err = nil
|
||||
return
|
||||
}
|
||||
|
||||
if currentFlags.Version {
|
||||
fmt.Println(version)
|
||||
return
|
||||
}
|
||||
|
||||
@ -95,6 +98,18 @@ func Cli() (message string, err error) {
|
||||
return
|
||||
}
|
||||
|
||||
// if the wipe context flag is set, run the wipe context function
|
||||
if currentFlags.WipeContext != "" {
|
||||
err = fabricDb.Contexts.Delete(currentFlags.WipeContext)
|
||||
return
|
||||
}
|
||||
|
||||
// if the wipe session flag is set, run the wipe session function
|
||||
if currentFlags.WipeSession != "" {
|
||||
err = fabricDb.Sessions.Delete(currentFlags.WipeSession)
|
||||
return
|
||||
}
|
||||
|
||||
// if the interactive flag is set, run the interactive function
|
||||
// if currentFlags.Interactive {
|
||||
// interactive.Interactive()
|
||||
@ -115,11 +130,15 @@ func Cli() (message string, err error) {
|
||||
|
||||
if !currentFlags.YouTubeComments || currentFlags.YouTubeTranscript {
|
||||
var transcript string
|
||||
if transcript, err = fabric.YouTube.GrabTranscript(videoId); err != nil {
|
||||
var language = "en"
|
||||
if currentFlags.Language != "" {
|
||||
language = currentFlags.Language
|
||||
}
|
||||
if transcript, err = fabric.YouTube.GrabTranscript(videoId, language); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
fmt.Println(transcript)
|
||||
// fmt.Println(transcript)
|
||||
|
||||
currentFlags.AppendMessage(transcript)
|
||||
}
|
||||
@ -132,13 +151,14 @@ func Cli() (message string, err error) {
|
||||
|
||||
commentsString := strings.Join(comments, "\n")
|
||||
|
||||
fmt.Println(commentsString)
|
||||
// fmt.Println(commentsString)
|
||||
|
||||
currentFlags.AppendMessage(commentsString)
|
||||
}
|
||||
|
||||
if currentFlags.Pattern == "" {
|
||||
// if the pattern flag is not set, we wanted only to grab the transcript or comments
|
||||
fmt.Println(currentFlags.Message)
|
||||
return
|
||||
}
|
||||
}
|
||||
@ -150,7 +170,7 @@ func Cli() (message string, err error) {
|
||||
return
|
||||
}
|
||||
|
||||
fmt.Println(message)
|
||||
//fmt.Println(message)
|
||||
|
||||
currentFlags.AppendMessage(message)
|
||||
}
|
||||
@ -161,13 +181,14 @@ func Cli() (message string, err error) {
|
||||
return
|
||||
}
|
||||
|
||||
fmt.Println(message)
|
||||
//fmt.Println(message)
|
||||
|
||||
currentFlags.AppendMessage(message)
|
||||
}
|
||||
|
||||
if currentFlags.Pattern == "" {
|
||||
// if the pattern flag is not set, we wanted only to grab the url or get the answer to the question
|
||||
fmt.Println(currentFlags.Message)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
@ -1,6 +1,7 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"github.com/danielmiessler/fabric/core"
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
@ -9,8 +10,14 @@ import (
|
||||
)
|
||||
|
||||
func TestCli(t *testing.T) {
|
||||
message, err := Cli()
|
||||
assert.NoError(t, err)
|
||||
t.Skip("Skipping test for now, collision with flag -t")
|
||||
originalArgs := os.Args
|
||||
defer func() { os.Args = originalArgs }()
|
||||
|
||||
os.Args = []string{os.Args[0]}
|
||||
message, err := Cli("test")
|
||||
assert.Error(t, err)
|
||||
assert.Equal(t, core.NoSessionPatternUserMessages, err.Error())
|
||||
assert.Empty(t, message)
|
||||
}
|
||||
|
||||
|
11
cli/flags.go
11
cli/flags.go
@ -37,13 +37,17 @@ type Flags struct {
|
||||
Output string `short:"o" long:"output" description:"Output to file" default:""`
|
||||
LatestPatterns string `short:"n" long:"latest" description:"Number of latest patterns to list" default:"0"`
|
||||
ChangeDefaultModel bool `short:"d" long:"changeDefaultModel" description:"Change default model"`
|
||||
YouTube string `short:"y" long:"youtube" description:"YouTube video url to grab transcript, comments from it and send to chat"`
|
||||
YouTubeTranscript bool `long:"transcript" description:"Grab transcript from YouTube video and send to chat"`
|
||||
YouTube string `short:"y" long:"youtube" description:"YouTube video \"URL\" to grab transcript, comments from it and send to chat"`
|
||||
YouTubeTranscript bool `long:"transcript" description:"Grab transcript from YouTube video and send to chat (it used per default)."`
|
||||
YouTubeComments bool `long:"comments" description:"Grab comments from YouTube video and send to chat"`
|
||||
DryRun bool `long:"dry-run" description:"Show what would be sent to the model without actually sending it"`
|
||||
Language string `short:"g" long:"language" description:"Specify the Language Code for the chat, e.g. -g=en -g=zh" default:""`
|
||||
ScrapeURL string `short:"u" long:"scrape_url" description:"Scrape website URL to markdown using Jina AI"`
|
||||
ScrapeQuestion string `short:"q" long:"scrape_question" description:"Search question using Jina AI"`
|
||||
Seed int `short:"e" long:"seed" description:"Seed to be used for LMM generation"`
|
||||
WipeContext string `short:"w" long:"wipecontext" description:"Wipe context"`
|
||||
WipeSession string `short:"W" long:"wipesession" description:"Wipe session"`
|
||||
DryRun bool `long:"dry-run" description:"Show what would be sent to the model without actually sending it"`
|
||||
Version bool `long:"version" description:"Print current version"`
|
||||
}
|
||||
|
||||
// Init Initialize flags. returns a Flags struct and an error
|
||||
@ -99,6 +103,7 @@ func (o *Flags) BuildChatOptions() (ret *common.ChatOptions) {
|
||||
PresencePenalty: o.PresencePenalty,
|
||||
FrequencyPenalty: o.FrequencyPenalty,
|
||||
Raw: o.Raw,
|
||||
Seed: o.Seed,
|
||||
}
|
||||
return
|
||||
}
|
||||
|
@ -53,6 +53,7 @@ func TestBuildChatOptions(t *testing.T) {
|
||||
TopP: 0.9,
|
||||
PresencePenalty: 0.1,
|
||||
FrequencyPenalty: 0.2,
|
||||
Seed: 1,
|
||||
}
|
||||
|
||||
expectedOptions := &common.ChatOptions{
|
||||
@ -61,6 +62,27 @@ func TestBuildChatOptions(t *testing.T) {
|
||||
PresencePenalty: 0.1,
|
||||
FrequencyPenalty: 0.2,
|
||||
Raw: false,
|
||||
Seed: 1,
|
||||
}
|
||||
options := flags.BuildChatOptions()
|
||||
assert.Equal(t, expectedOptions, options)
|
||||
}
|
||||
|
||||
func TestBuildChatOptionsDefaultSeed(t *testing.T) {
|
||||
flags := &Flags{
|
||||
Temperature: 0.8,
|
||||
TopP: 0.9,
|
||||
PresencePenalty: 0.1,
|
||||
FrequencyPenalty: 0.2,
|
||||
}
|
||||
|
||||
expectedOptions := &common.ChatOptions{
|
||||
Temperature: 0.8,
|
||||
TopP: 0.9,
|
||||
PresencePenalty: 0.1,
|
||||
FrequencyPenalty: 0.2,
|
||||
Raw: false,
|
||||
Seed: 0,
|
||||
}
|
||||
options := flags.BuildChatOptions()
|
||||
assert.Equal(t, expectedOptions, options)
|
||||
|
@ -103,8 +103,9 @@ func (o *Setting) IsDefined() bool {
|
||||
}
|
||||
|
||||
func (o *Setting) Configure() error {
|
||||
if o.Value == "" {
|
||||
o.Value = os.Getenv(o.EnvVariable)
|
||||
envValue := os.Getenv(o.EnvVariable)
|
||||
if envValue != "" {
|
||||
o.Value = envValue
|
||||
}
|
||||
return o.IsValidErr()
|
||||
}
|
||||
|
@ -23,6 +23,7 @@ type ChatOptions struct {
|
||||
PresencePenalty float64
|
||||
FrequencyPenalty float64
|
||||
Raw bool
|
||||
Seed int
|
||||
}
|
||||
|
||||
// NormalizeMessages remove empty messages and ensure messages order user-assist-user
|
||||
|
@ -30,6 +30,8 @@ import (
|
||||
const DefaultPatternsGitRepoUrl = "https://github.com/danielmiessler/fabric.git"
|
||||
const DefaultPatternsGitRepoFolder = "patterns"
|
||||
|
||||
const NoSessionPatternUserMessages = "no session, pattern or user messages provided"
|
||||
|
||||
func NewFabric(db *db.Db) (ret *Fabric, err error) {
|
||||
ret = NewFabricBase(db)
|
||||
err = ret.Configure()
|
||||
@ -279,7 +281,7 @@ func (o *Chat) BuildChatSession(raw bool) (ret *db.Session, err error) {
|
||||
|
||||
if ret.IsEmpty() {
|
||||
ret = nil
|
||||
err = fmt.Errorf("no session, pattern or user messages provided")
|
||||
err = fmt.Errorf(NoSessionPatternUserMessages)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
2
go.mod
2
go.mod
@ -16,6 +16,7 @@ require (
|
||||
github.com/samber/lo v1.47.0
|
||||
github.com/sashabaranov/go-openai v1.30.0
|
||||
github.com/stretchr/testify v1.9.0
|
||||
golang.org/x/text v0.18.0
|
||||
google.golang.org/api v0.197.0
|
||||
)
|
||||
|
||||
@ -61,7 +62,6 @@ require (
|
||||
golang.org/x/oauth2 v0.23.0 // indirect
|
||||
golang.org/x/sync v0.8.0 // indirect
|
||||
golang.org/x/sys v0.25.0 // indirect
|
||||
golang.org/x/text v0.18.0 // indirect
|
||||
golang.org/x/time v0.6.0 // indirect
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20240903143218-8af14fe29dc1 // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect
|
||||
|
5
main.go
5
main.go
@ -2,14 +2,15 @@ package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/jessevdk/go-flags"
|
||||
"os"
|
||||
|
||||
"github.com/danielmiessler/fabric/cli"
|
||||
)
|
||||
|
||||
func main() {
|
||||
_, err := cli.Cli()
|
||||
if err != nil {
|
||||
_, err := cli.Cli(version)
|
||||
if err != nil && !flags.WroteHelp(err) {
|
||||
fmt.Printf("%s\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
14
vendors/openai/openai.go
vendored
14
vendors/openai/openai.go
vendored
@ -5,6 +5,7 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
|
||||
"github.com/danielmiessler/fabric/common"
|
||||
"github.com/samber/lo"
|
||||
@ -111,6 +112,7 @@ func (o *Client) Send(ctx context.Context, msgs []*common.Message, opts *common.
|
||||
}
|
||||
if len(resp.Choices) > 0 {
|
||||
ret = resp.Choices[0].Message.Content
|
||||
slog.Debug("SystemFingerprint: " + resp.SystemFingerprint)
|
||||
}
|
||||
return
|
||||
}
|
||||
@ -128,6 +130,7 @@ func (o *Client) buildChatCompletionRequest(
|
||||
Messages: messages,
|
||||
}
|
||||
} else {
|
||||
if opts.Seed == 0 {
|
||||
ret = goopenai.ChatCompletionRequest{
|
||||
Model: opts.Model,
|
||||
Temperature: float32(opts.Temperature),
|
||||
@ -136,6 +139,17 @@ func (o *Client) buildChatCompletionRequest(
|
||||
FrequencyPenalty: float32(opts.FrequencyPenalty),
|
||||
Messages: messages,
|
||||
}
|
||||
} else {
|
||||
ret = goopenai.ChatCompletionRequest{
|
||||
Model: opts.Model,
|
||||
Temperature: float32(opts.Temperature),
|
||||
TopP: float32(opts.TopP),
|
||||
PresencePenalty: float32(opts.PresencePenalty),
|
||||
FrequencyPenalty: float32(opts.FrequencyPenalty),
|
||||
Messages: messages,
|
||||
Seed: &opts.Seed,
|
||||
}
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
102
vendors/openai/openai_test.go
vendored
Normal file
102
vendors/openai/openai_test.go
vendored
Normal file
@ -0,0 +1,102 @@
|
||||
package openai
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/danielmiessler/fabric/common"
|
||||
"github.com/sashabaranov/go-openai"
|
||||
goopenai "github.com/sashabaranov/go-openai"
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestBuildChatCompletionRequestPinSeed(t *testing.T) {
|
||||
|
||||
var msgs []*common.Message
|
||||
|
||||
for i := 0; i < 2; i++ {
|
||||
msgs = append(msgs, &common.Message{
|
||||
Role: "User",
|
||||
Content: "My msg",
|
||||
})
|
||||
}
|
||||
|
||||
opts := &common.ChatOptions{
|
||||
Temperature: 0.8,
|
||||
TopP: 0.9,
|
||||
PresencePenalty: 0.1,
|
||||
FrequencyPenalty: 0.2,
|
||||
Raw: false,
|
||||
Seed: 1,
|
||||
}
|
||||
|
||||
var expectedMessages []openai.ChatCompletionMessage
|
||||
|
||||
for i := 0; i < 2; i++ {
|
||||
expectedMessages = append(expectedMessages,
|
||||
openai.ChatCompletionMessage{
|
||||
Role: msgs[i].Role,
|
||||
Content: msgs[i].Content,
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
var expectedRequest = goopenai.ChatCompletionRequest{
|
||||
Model: opts.Model,
|
||||
Temperature: float32(opts.Temperature),
|
||||
TopP: float32(opts.TopP),
|
||||
PresencePenalty: float32(opts.PresencePenalty),
|
||||
FrequencyPenalty: float32(opts.FrequencyPenalty),
|
||||
Messages: expectedMessages,
|
||||
Seed: &opts.Seed,
|
||||
}
|
||||
|
||||
var client = NewClient()
|
||||
request := client.buildChatCompletionRequest(msgs, opts)
|
||||
assert.Equal(t, expectedRequest, request)
|
||||
}
|
||||
|
||||
func TestBuildChatCompletionRequestNilSeed(t *testing.T) {
|
||||
|
||||
var msgs []*common.Message
|
||||
|
||||
for i := 0; i < 2; i++ {
|
||||
msgs = append(msgs, &common.Message{
|
||||
Role: "User",
|
||||
Content: "My msg",
|
||||
})
|
||||
}
|
||||
|
||||
opts := &common.ChatOptions{
|
||||
Temperature: 0.8,
|
||||
TopP: 0.9,
|
||||
PresencePenalty: 0.1,
|
||||
FrequencyPenalty: 0.2,
|
||||
Raw: false,
|
||||
Seed: 0,
|
||||
}
|
||||
|
||||
var expectedMessages []openai.ChatCompletionMessage
|
||||
|
||||
for i := 0; i < 2; i++ {
|
||||
expectedMessages = append(expectedMessages,
|
||||
openai.ChatCompletionMessage{
|
||||
Role: msgs[i].Role,
|
||||
Content: msgs[i].Content,
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
var expectedRequest = goopenai.ChatCompletionRequest{
|
||||
Model: opts.Model,
|
||||
Temperature: float32(opts.Temperature),
|
||||
TopP: float32(opts.TopP),
|
||||
PresencePenalty: float32(opts.PresencePenalty),
|
||||
FrequencyPenalty: float32(opts.FrequencyPenalty),
|
||||
Messages: expectedMessages,
|
||||
Seed: nil,
|
||||
}
|
||||
|
||||
var client = NewClient()
|
||||
request := client.buildChatCompletionRequest(msgs, opts)
|
||||
assert.Equal(t, expectedRequest, request)
|
||||
}
|
3
version.go
Normal file
3
version.go
Normal file
@ -0,0 +1,3 @@
|
||||
package main
|
||||
|
||||
var version = "v1.4.31-5d3e0da"
|
@ -10,6 +10,7 @@ import (
|
||||
"google.golang.org/api/option"
|
||||
"google.golang.org/api/youtube/v3"
|
||||
"log"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
@ -61,17 +62,17 @@ func (o *YouTube) GetVideoId(url string) (ret string, err error) {
|
||||
return
|
||||
}
|
||||
|
||||
func (o *YouTube) GrabTranscriptForUrl(url string) (ret string, err error) {
|
||||
func (o *YouTube) GrabTranscriptForUrl(url string, language string) (ret string, err error) {
|
||||
var videoId string
|
||||
if videoId, err = o.GetVideoId(url); err != nil {
|
||||
return
|
||||
}
|
||||
return o.GrabTranscript(videoId)
|
||||
return o.GrabTranscript(videoId, language)
|
||||
}
|
||||
|
||||
func (o *YouTube) GrabTranscript(videoId string) (ret string, err error) {
|
||||
func (o *YouTube) GrabTranscript(videoId string, language string) (ret string, err error) {
|
||||
var transcript string
|
||||
if transcript, err = o.GrabTranscriptBase(videoId); err != nil {
|
||||
if transcript, err = o.GrabTranscriptBase(videoId, language); err != nil {
|
||||
err = fmt.Errorf("transcript not available. (%v)", err)
|
||||
return
|
||||
}
|
||||
@ -89,14 +90,14 @@ func (o *YouTube) GrabTranscript(videoId string) (ret string, err error) {
|
||||
return
|
||||
}
|
||||
|
||||
func (o *YouTube) GrabTranscriptBase(videoId string) (ret string, err error) {
|
||||
func (o *YouTube) GrabTranscriptBase(videoId string, language string) (ret string, err error) {
|
||||
if err = o.initService(); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
url := "https://www.youtube.com/watch?v=" + videoId
|
||||
watchUrl := "https://www.youtube.com/watch?v=" + videoId
|
||||
var resp string
|
||||
if resp, err = soup.Get(url); err != nil {
|
||||
if resp, err = soup.Get(watchUrl); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
@ -117,6 +118,16 @@ func (o *YouTube) GrabTranscriptBase(videoId string) (ret string, err error) {
|
||||
|
||||
if len(captionTracks) > 0 {
|
||||
transcriptURL := captionTracks[0].BaseURL
|
||||
for _, captionTrack := range captionTracks {
|
||||
parsedUrl, error := url.Parse(captionTrack.BaseURL)
|
||||
if error != nil {
|
||||
err = fmt.Errorf("error parsing caption track")
|
||||
}
|
||||
parsedUrlParams, _ := url.ParseQuery(parsedUrl.RawQuery)
|
||||
if parsedUrlParams["lang"][0] == language {
|
||||
transcriptURL = captionTrack.BaseURL
|
||||
}
|
||||
}
|
||||
ret, err = soup.Get(transcriptURL)
|
||||
return
|
||||
}
|
||||
@ -212,7 +223,7 @@ func (o *YouTube) Grab(url string, options *Options) (ret *VideoInfo, err error)
|
||||
}
|
||||
|
||||
if options.Transcript {
|
||||
if ret.Transcript, err = o.GrabTranscript(videoId); err != nil {
|
||||
if ret.Transcript, err = o.GrabTranscript(videoId, "en"); err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
|
Loading…
x
Reference in New Issue
Block a user