Зафиксирована рабочая версия TEN-Agent для HuggingFace Space

.devcontainer/devcontainer.json

@@ -0,0 +1,28 @@
+// For format details, see https://aka.ms/devcontainer.json. For config options, see the
+// README at: https://github.com/devcontainers/templates/tree/main/src/docker-existing-dockerfile
+{
+    "name": "ten_agent_dev",
+    "image": "ghcr.io/ten-framework/ten_agent_build:0.4.17",
+    "customizations": {
+        "vscode": {
+            "extensions": [
+                "golang.go",
+                "ms-vscode.cpptools"
+            ]
+        }
+    },
+    "workspaceMount": "source=${localWorkspaceFolder},target=/app,type=bind",
+    "workspaceFolder": "/app",
+    // Use 'forwardPorts' to make a list of ports inside the container available locally.
+    "forwardPorts": [
+        3000,
+        8080,
+        49483
+    ],
+    // Features to add to the dev container. More info: https://containers.dev/features.
+    "features": {
+        "ghcr.io/devcontainers/features/git:1": {},
+        "ghcr.io/devcontainers/features/python:1": {},
+        "ghcr.io/devcontainers/features/node:1": {}
+    }
+}

.dockerignore

@@ -0,0 +1,2 @@
+.git
+playground/

.gitattributes

@@ -0,0 +1,5 @@
+# Shell scripts use LF as line separator, even checked out to Windows(NTFS) file-system
+*.sh text eol=lf
+agents/bin/* text eol=lf
+agents/scripts/* text eol=lf
+*.lockb filter=lfs diff=lfs merge=lfs -text

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -0,0 +1,67 @@
+name: Bug Report
+description: Report a bug or issue with the project
+title: "[BUG] "
+labels: [bug]
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Thanks for taking the time to report a bug! Please fill in the following details.
+
+  - type: textarea
+    id: description
+    attributes:
+      label: Description
+      description: A clear and detailed description of the bug.
+      placeholder: "Enter a clear and concise description of what the bug is."
+    validations:
+      required: true
+
+  - type: input
+    id: environment
+    attributes:
+      label: Environment
+      description: The environment where this bug occurred (e.g., operating system, CPU arch, etc.).
+      placeholder: "Enter details about the environment."
+    validations:
+      required: true
+
+  - type: textarea
+    id: steps
+    attributes:
+      label: Steps to reproduce
+      description: What are the steps to reproduce this issue?
+      placeholder: |
+        1. ...
+    validations:
+      required: true
+
+  - type: textarea
+    id: expected
+    attributes:
+      label: Expected behavior
+      description: What should have happened instead?
+      placeholder: "Describe what you expected to happen."
+    validations:
+      required: true
+
+  - type: dropdown
+    id: severity
+    attributes:
+      label: Severity
+      description: How severe is the bug?
+      options:
+        - Critical
+        - Major
+        - Minor
+    validations:
+      required: true
+
+  - type: textarea
+    id: additional_info
+    attributes:
+      label: Additional Information
+      description: Any other context or screenshots related to the bug.
+      placeholder: "Enter additional context or information."
+    validations:
+      required: false

.github/ISSUE_TEMPLATE/feature_request.yml

@@ -0,0 +1,39 @@
+name: Feature Request
+description: Request a feature
+title: "[FEATURE] "
+labels: [feature]
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Thanks for taking the time to request a feature! Please fill in the following details.
+
+  - type: textarea
+    id: description
+    attributes:
+      label: Description
+      description: A clear and detailed description of the feature request.
+      placeholder: "Enter a clear and concise description of what the feature request is."
+    validations:
+      required: true
+
+  - type: dropdown
+    id: severity
+    attributes:
+      label: Severity
+      description: How severe is the bug?
+      options:
+        - Critical
+        - Major
+        - Minor
+    validations:
+      required: true
+
+  - type: textarea
+    id: additional_info
+    attributes:
+      label: Additional Information
+      description: Any other context or screenshots related to the bug.
+      placeholder: "Enter additional context or information."
+    validations:
+      required: false

.github/workflows/build-docker.yaml

@@ -0,0 +1,87 @@
+name: Build Docker
+
+on:
+  push:
+    branches: [ "**" ]
+    tags: [ "**" ]
+    paths-ignore:
+      - ".devcontainer/**"
+      - ".github/**"
+      - "!.github/workflows/build-docker.yaml"
+      - ".vscode/**"
+      - "docs/**"
+      - "**.md"
+  pull_request:
+    branches: [ "main" ]
+    paths-ignore:
+      - ".devcontainer/**"
+      - ".github/**"
+      - "!.github/workflows/build-docker.yaml"
+      - ".vscode/**"
+      - "docs/**"
+      - "**.md"
+  workflow_dispatch:
+
+env:
+  SERVER_IMAGE_NAME: ten_agent_server
+  PLAYGROUND_IMAGE_NAME: ten_agent_playground
+  NON_EDIT_PLAYGROUND_IMAGE_NAME: ten_agent_non_edit_playground
+  DEMO_IMAGE_NAME: ten_agent_demo
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-tags: true
+          fetch-depth: "0"
+      - id: pre-step
+        shell: bash
+        run: echo "image-tag=$(git describe --tags --always)" >> $GITHUB_OUTPUT
+      - name: Build & Publish Docker Image for Agents Server
+        uses: elgohr/Publish-Docker-Github-Action@v5
+        with:
+          name: ${{ github.repository_owner }}/${{ env.SERVER_IMAGE_NAME }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+          registry: ghcr.io
+          tags: "${{ github.ref == 'refs/heads/main' && 'latest,' || '' }}${{ steps.pre-step.outputs.image-tag }}"
+          no_push: ${{ github.event_name == 'pull_request' }}
+      - name: Build & Publish Docker Image for Playground
+        uses: elgohr/Publish-Docker-Github-Action@v5
+        env:
+          EDIT_GRAPH_MODE: true
+        with:
+          name: ${{ github.repository_owner }}/${{ env.PLAYGROUND_IMAGE_NAME }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+          registry: ghcr.io
+          workdir: playground
+          tags: "${{ github.ref == 'refs/heads/main' && 'latest,' || '' }}${{ steps.pre-step.outputs.image-tag }}"
+          no_push: ${{ github.event_name == 'pull_request' }}
+          buildargs: EDIT_GRAPH_MODE
+      - name: Build & Publish Docker Image for Non-Editable Playground
+        uses: elgohr/Publish-Docker-Github-Action@v5
+        env:
+          EDIT_GRAPH_MODE: false
+        with:
+          name: ${{ github.repository_owner }}/${{ env.NON_EDIT_PLAYGROUND_IMAGE_NAME }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+          registry: ghcr.io
+          workdir: playground
+          tags: "${{ github.ref == 'refs/heads/main' && 'latest,' || '' }}${{ steps.pre-step.outputs.image-tag }}"
+          no_push: ${{ github.event_name == 'pull_request' }}
+          buildargs: EDIT_GRAPH_MODE
+      - name: Build & Publish Docker Image for demo
+        uses: elgohr/Publish-Docker-Github-Action@v5
+        with:
+          name: ${{ github.repository_owner }}/${{ env.DEMO_IMAGE_NAME }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+          registry: ghcr.io
+          workdir: demo
+          tags: "${{ github.ref == 'refs/heads/main' && 'latest,' || '' }}${{ steps.pre-step.outputs.image-tag }}"
+          no_push: ${{ github.event_name == 'pull_request' }}

.github/workflows/ci.yaml

@@ -0,0 +1,45 @@
+name: CI
+
+on:
+  pull_request:
+    branches: [ "main" ]
+    paths-ignore:
+      - ".devcontainer/**"
+      - ".github/**"
+      - "!.github/workflows/ci.yaml"
+      - ".vscode/**"
+      - "docs/**"
+      - "esp32-client/**"
+      - "**.md"
+      - "Dockerfile"
+      - "docker-compose.yml"
+      - "demo/**"
+      - "playground/**"
+  workflow_dispatch:
+
+jobs:
+  ci:
+    runs-on: ubuntu-latest
+    container:
+      image: ghcr.io/ten-framework/ten_agent_build:0.4.17
+    strategy:
+      matrix:
+        agent: [agents/examples/default, agents/examples/demo, agents/examples/experimental]
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: "0"
+          submodules: "true"
+
+      - name: Use agent
+        run: |
+          git config --global --add safe.directory $(pwd)
+          task use AGENT=${{ matrix.agent }}
+
+      - name: Run tests
+        run: |
+          task test -- -s -v
+
+      # - name: Run lint
+      #   run: |
+      #     task lint

.pylintrc

@@ -0,0 +1,652 @@
+[MAIN]
+
+# Analyse import fallback blocks. This can be used to support both Python 2 and
+# 3 compatible code, which means that the block might have code that exists
+# only in one or another interpreter, leading to false positives when analysed.
+analyse-fallback-blocks=no
+
+# Clear in-memory caches upon conclusion of linting. Useful if running pylint
+# in a server-like mode.
+clear-cache-post-run=no
+
+# Load and enable all available extensions. Use --list-extensions to see a list
+# all available extensions.
+#enable-all-extensions=
+
+# In error mode, messages with a category besides ERROR or FATAL are
+# suppressed, and no reports are done by default. Error mode is compatible with
+# disabling specific errors.
+#errors-only=
+
+# Always return a 0 (non-error) status code, even if lint errors are found.
+# This is primarily useful in continuous integration scripts.
+#exit-zero=
+
+# A comma-separated list of package or module names from where C extensions may
+# be loaded. Extensions are loading into the active Python interpreter and may
+# run arbitrary code.
+extension-pkg-allow-list=
+
+# A comma-separated list of package or module names from where C extensions may
+# be loaded. Extensions are loading into the active Python interpreter and may
+# run arbitrary code. (This is an alternative name to extension-pkg-allow-list
+# for backward compatibility.)
+extension-pkg-whitelist=
+
+# Return non-zero exit code if any of these messages/categories are detected,
+# even if score is above --fail-under value. Syntax same as enable. Messages
+# specified are enabled, while categories only check already-enabled messages.
+fail-on=
+
+# Specify a score threshold under which the program will exit with error.
+fail-under=10
+
+# Interpret the stdin as a python script, whose filename needs to be passed as
+# the module_or_package argument.
+#from-stdin=
+
+# Files or directories to be skipped. They should be base names, not paths.
+ignore=CVS,examples,tests,out
+
+# Add files or directories matching the regular expressions patterns to the
+# ignore-list. The regex matches against paths and can be in Posix or Windows
+# format. Because '\\' represents the directory delimiter on Windows systems,
+# it can't be used as an escape character.
+ignore-paths=
+
+# Files or directories matching the regular expression patterns are skipped.
+# The regex matches against base names, not paths. The default value ignores
+# Emacs file locks
+ignore-patterns=
+
+# List of module names for which member attributes should not be checked and
+# will not be imported (useful for modules/projects where namespaces are
+# manipulated during runtime and thus existing member attributes cannot be
+# deduced by static analysis). It supports qualified module names, as well as
+# Unix pattern matching.
+ignored-modules=
+
+# Python code to execute, usually for sys.path manipulation such as
+# pygtk.require().
+#init-hook=
+
+# Use multiple processes to speed up Pylint. Specifying 0 will auto-detect the
+# number of processors available to use, and will cap the count on Windows to
+# avoid hangs.
+jobs=1
+
+# Control the amount of potential inferred values when inferring a single
+# object. This can help the performance when dealing with large functions or
+# complex, nested conditions.
+limit-inference-results=100
+
+# List of plugins (as comma separated values of python module names) to load,
+# usually to register additional checkers.
+load-plugins=
+
+# Pickle collected data for later comparisons.
+persistent=yes
+
+# Resolve imports to .pyi stubs if available. May reduce no-member messages and
+# increase not-an-iterable messages.
+prefer-stubs=no
+
+# Minimum Python version to use for version dependent checks. Will default to
+# the version used to run pylint.
+py-version=3.10
+
+# Discover python modules and packages in the file system subtree.
+recursive=no
+
+# Add paths to the list of the source roots. Supports globbing patterns. The
+# source root is an absolute path or a path relative to the current working
+# directory used to determine a package namespace for modules located under the
+# source root.
+source-roots=
+
+# When enabled, pylint would attempt to guess common misconfiguration and emit
+# user-friendly hints instead of false-positive error messages.
+suggestion-mode=yes
+
+# Allow loading of arbitrary C extensions. Extensions are imported into the
+# active Python interpreter and may run arbitrary code.
+unsafe-load-any-extension=no
+
+# In verbose mode, extra non-checker-related info will be displayed.
+#verbose=
+
+
+[BASIC]
+
+# Naming style matching correct argument names.
+argument-naming-style=snake_case
+
+# Regular expression matching correct argument names. Overrides argument-
+# naming-style. If left empty, argument names will be checked with the set
+# naming style.
+#argument-rgx=
+
+# Naming style matching correct attribute names.
+attr-naming-style=snake_case
+
+# Regular expression matching correct attribute names. Overrides attr-naming-
+# style. If left empty, attribute names will be checked with the set naming
+# style.
+#attr-rgx=
+
+# Bad variable names which should always be refused, separated by a comma.
+bad-names=foo,
+          bar,
+          baz,
+          toto,
+          tutu,
+          tata
+
+# Bad variable names regexes, separated by a comma. If names match any regex,
+# they will always be refused
+bad-names-rgxs=
+
+# Naming style matching correct class attribute names.
+class-attribute-naming-style=any
+
+# Regular expression matching correct class attribute names. Overrides class-
+# attribute-naming-style. If left empty, class attribute names will be checked
+# with the set naming style.
+#class-attribute-rgx=
+
+# Naming style matching correct class constant names.
+class-const-naming-style=UPPER_CASE
+
+# Regular expression matching correct class constant names. Overrides class-
+# const-naming-style. If left empty, class constant names will be checked with
+# the set naming style.
+#class-const-rgx=
+
+# Naming style matching correct class names.
+class-naming-style=PascalCase
+
+# Regular expression matching correct class names. Overrides class-naming-
+# style. If left empty, class names will be checked with the set naming style.
+#class-rgx=
+
+# Naming style matching correct constant names.
+const-naming-style=UPPER_CASE
+
+# Regular expression matching correct constant names. Overrides const-naming-
+# style. If left empty, constant names will be checked with the set naming
+# style.
+#const-rgx=
+
+# Minimum line length for functions/classes that require docstrings, shorter
+# ones are exempt.
+docstring-min-length=-1
+
+# Naming style matching correct function names.
+function-naming-style=snake_case
+
+# Regular expression matching correct function names. Overrides function-
+# naming-style. If left empty, function names will be checked with the set
+# naming style.
+#function-rgx=
+
+# Good variable names which should always be accepted, separated by a comma.
+good-names=i,
+           j,
+           k,
+           ex,
+           Run,
+           _
+
+# Good variable names regexes, separated by a comma. If names match any regex,
+# they will always be accepted
+good-names-rgxs=
+
+# Include a hint for the correct naming format with invalid-name.
+include-naming-hint=no
+
+# Naming style matching correct inline iteration names.
+inlinevar-naming-style=any
+
+# Regular expression matching correct inline iteration names. Overrides
+# inlinevar-naming-style. If left empty, inline iteration names will be checked
+# with the set naming style.
+#inlinevar-rgx=
+
+# Naming style matching correct method names.
+method-naming-style=snake_case
+
+# Regular expression matching correct method names. Overrides method-naming-
+# style. If left empty, method names will be checked with the set naming style.
+#method-rgx=
+
+# Naming style matching correct module names.
+module-naming-style=snake_case
+
+# Regular expression matching correct module names. Overrides module-naming-
+# style. If left empty, module names will be checked with the set naming style.
+#module-rgx=
+
+# Colon-delimited sets of names that determine each other's naming style when
+# the name regexes allow several styles.
+name-group=
+
+# Regular expression which should only match function or class names that do
+# not require a docstring.
+no-docstring-rgx=^_
+
+# List of decorators that produce properties, such as abc.abstractproperty. Add
+# to this list to register other decorators that produce valid properties.
+# These decorators are taken in consideration only for invalid-name.
+property-classes=abc.abstractproperty
+
+# Regular expression matching correct type alias names. If left empty, type
+# alias names will be checked with the set naming style.
+#typealias-rgx=
+
+# Regular expression matching correct type variable names. If left empty, type
+# variable names will be checked with the set naming style.
+#typevar-rgx=
+
+# Naming style matching correct variable names.
+variable-naming-style=snake_case
+
+# Regular expression matching correct variable names. Overrides variable-
+# naming-style. If left empty, variable names will be checked with the set
+# naming style.
+#variable-rgx=
+
+
+[CLASSES]
+
+# Warn about protected attribute access inside special methods
+check-protected-access-in-special-methods=no
+
+# List of method names used to declare (i.e. assign) instance attributes.
+defining-attr-methods=__init__,
+                      __new__,
+                      setUp,
+                      asyncSetUp,
+                      __post_init__
+
+# List of member names, which should be excluded from the protected access
+# warning.
+exclude-protected=_asdict,_fields,_replace,_source,_make,os._exit
+
+# List of valid names for the first argument in a class method.
+valid-classmethod-first-arg=cls
+
+# List of valid names for the first argument in a metaclass class method.
+valid-metaclass-classmethod-first-arg=mcs
+
+
+[DESIGN]
+
+# List of regular expressions of class ancestor names to ignore when counting
+# public methods (see R0903)
+exclude-too-few-public-methods=
+
+# List of qualified class names to ignore when counting class parents (see
+# R0901)
+ignored-parents=
+
+# Maximum number of arguments for function / method.
+max-args=5
+
+# Maximum number of attributes for a class (see R0902).
+max-attributes=7
+
+# Maximum number of boolean expressions in an if statement (see R0916).
+max-bool-expr=5
+
+# Maximum number of branch for function / method body.
+max-branches=12
+
+# Maximum number of locals for function / method body.
+max-locals=15
+
+# Maximum number of parents for a class (see R0901).
+max-parents=7
+
+# Maximum number of positional arguments for function / method.
+#max-positional-arguments=5
+
+# Maximum number of public methods for a class (see R0904).
+max-public-methods=20
+
+# Maximum number of return / yield for function / method body.
+max-returns=6
+
+# Maximum number of statements in function / method body.
+max-statements=50
+
+# Minimum number of public methods for a class (see R0903).
+min-public-methods=2
+
+
+[EXCEPTIONS]
+
+# Exceptions that will emit a warning when caught.
+overgeneral-exceptions=builtins.BaseException,builtins.Exception
+
+
+[FORMAT]
+
+# Expected format of line ending, e.g. empty (any line ending), LF or CRLF.
+expected-line-ending-format=
+
+# Regexp for a line that is allowed to be longer than the limit.
+ignore-long-lines=^\s*(# )?<?https?://\S+>?$
+
+# Number of spaces of indent required inside a hanging or continued line.
+indent-after-paren=4
+
+# String used as indentation unit. This is usually "    " (4 spaces) or "\t" (1
+# tab).
+indent-string='    '
+
+# Maximum number of characters on a single line.
+max-line-length=100
+
+# Maximum number of lines in a module.
+max-module-lines=1000
+
+# Allow the body of a class to be on the same line as the declaration if body
+# contains single statement.
+single-line-class-stmt=no
+
+# Allow the body of an if to be on the same line as the test if there is no
+# else.
+single-line-if-stmt=no
+
+
+[IMPORTS]
+
+# List of modules that can be imported at any level, not just the top level
+# one.
+allow-any-import-level=
+
+# Allow explicit reexports by alias from a package __init__.
+allow-reexport-from-package=no
+
+# Allow wildcard imports from modules that define __all__.
+allow-wildcard-with-all=no
+
+# Deprecated modules which should not be used, separated by a comma.
+deprecated-modules=
+
+# Output a graph (.gv or any supported image format) of external dependencies
+# to the given file (report RP0402 must not be disabled).
+ext-import-graph=
+
+# Output a graph (.gv or any supported image format) of all (i.e. internal and
+# external) dependencies to the given file (report RP0402 must not be
+# disabled).
+import-graph=
+
+# Output a graph (.gv or any supported image format) of internal dependencies
+# to the given file (report RP0402 must not be disabled).
+int-import-graph=
+
+# Force import order to recognize a module as part of the standard
+# compatibility libraries.
+known-standard-library=
+
+# Force import order to recognize a module as part of a third party library.
+known-third-party=enchant
+
+# Couples of modules and preferred modules, separated by a comma.
+preferred-modules=
+
+
+[LOGGING]
+
+# The type of string formatting that logging methods do. `old` means using %
+# formatting, `new` is for `{}` formatting.
+logging-format-style=old
+
+# Logging modules to check that the string format arguments are in logging
+# function parameter format.
+logging-modules=logging
+
+
+[MESSAGES CONTROL]
+
+# Only show warnings with the listed confidence levels. Leave empty to show
+# all. Valid levels: HIGH, CONTROL_FLOW, INFERENCE, INFERENCE_FAILURE,
+# UNDEFINED.
+confidence=HIGH,
+           CONTROL_FLOW,
+           INFERENCE,
+           INFERENCE_FAILURE,
+           UNDEFINED
+
+# Disable the message, report, category or checker with the given id(s). You
+# can either give multiple identifiers separated by comma (,) or put this
+# option multiple times (only on the command line, not in the configuration
+# file where it should appear only once). You can also use "--disable=all" to
+# disable everything first and then re-enable specific checks. For example, if
+# you want to run only the similarities checker, you can use "--disable=all
+# --enable=similarities". If you want to run only the classes checker, but have
+# no Warning level messages displayed, use "--disable=all --enable=classes
+# --disable=W".
+disable=raw-checker-failed,
+        bad-inline-option,
+        locally-disabled,
+        file-ignored,
+        suppressed-message,
+        useless-suppression,
+        deprecated-pragma,
+        use-symbolic-message-instead,
+        use-implicit-booleaness-not-comparison-to-string,
+        use-implicit-booleaness-not-comparison-to-zero,
+        broad-exception-caught,
+        logging-fstring-interpolation,
+        arguments-renamed,
+        I,C,R,
+        fixme,
+
+# Enable the message, report, category or checker with the given id(s). You can
+# either give multiple identifier separated by comma (,) or put this option
+# multiple time (only on the command line, not in the configuration file where
+# it should appear only once). See also the "--disable" option for examples.
+enable=
+
+
+[METHOD_ARGS]
+
+# List of qualified names (i.e., library.method) which require a timeout
+# parameter e.g. 'requests.api.get,requests.api.post'
+timeout-methods=requests.api.delete,requests.api.get,requests.api.head,requests.api.options,requests.api.patch,requests.api.post,requests.api.put,requests.api.request
+
+
+[MISCELLANEOUS]
+
+# List of note tags to take in consideration, separated by a comma.
+notes=FIXME,
+      XXX,
+      TODO
+
+# Regular expression of note tags to take in consideration.
+notes-rgx=
+
+
+[REFACTORING]
+
+# Maximum number of nested blocks for function / method body
+max-nested-blocks=5
+
+# Complete name of functions that never returns. When checking for
+# inconsistent-return-statements if a never returning function is called then
+# it will be considered as an explicit return statement and no message will be
+# printed.
+never-returning-functions=sys.exit,argparse.parse_error
+
+# Let 'consider-using-join' be raised when the separator to join on would be
+# non-empty (resulting in expected fixes of the type: ``"- " + " -
+# ".join(items)``)
+suggest-join-with-non-empty-separator=yes
+
+
+[REPORTS]
+
+# Python expression which should return a score less than or equal to 10. You
+# have access to the variables 'fatal', 'error', 'warning', 'refactor',
+# 'convention', and 'info' which contain the number of messages in each
+# category, as well as 'statement' which is the total number of statements
+# analyzed. This score is used by the global evaluation report (RP0004).
+evaluation=max(0, 0 if fatal else 10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10))
+
+# Template used to display messages. This is a python new-style format string
+# used to format the message information. See doc for all details.
+msg-template=
+
+# Set the output format. Available formats are: text, parseable, colorized,
+# json2 (improved json format), json (old json format) and msvs (visual
+# studio). You can also give a reporter class, e.g.
+# mypackage.mymodule.MyReporterClass.
+#output-format=
+
+# Tells whether to display a full report or only the messages.
+reports=no
+
+# Activate the evaluation score.
+score=yes
+
+
+[SIMILARITIES]
+
+# Comments are removed from the similarity computation
+ignore-comments=yes
+
+# Docstrings are removed from the similarity computation
+ignore-docstrings=yes
+
+# Imports are removed from the similarity computation
+ignore-imports=yes
+
+# Signatures are removed from the similarity computation
+ignore-signatures=yes
+
+# Minimum lines number of a similarity.
+min-similarity-lines=4
+
+
+[SPELLING]
+
+# Limits count of emitted suggestions for spelling mistakes.
+max-spelling-suggestions=4
+
+# Spelling dictionary name. No available dictionaries : You need to install
+# both the python package and the system dependency for enchant to work.
+spelling-dict=
+
+# List of comma separated words that should be considered directives if they
+# appear at the beginning of a comment and should not be checked.
+spelling-ignore-comment-directives=fmt: on,fmt: off,noqa:,noqa,nosec,isort:skip,mypy:
+
+# List of comma separated words that should not be checked.
+spelling-ignore-words=
+
+# A path to a file that contains the private dictionary; one word per line.
+spelling-private-dict-file=
+
+# Tells whether to store unknown words to the private dictionary (see the
+# --spelling-private-dict-file option) instead of raising a message.
+spelling-store-unknown-words=no
+
+
+[STRING]
+
+# This flag controls whether inconsistent-quotes generates a warning when the
+# character used as a quote delimiter is used inconsistently within a module.
+check-quote-consistency=no
+
+# This flag controls whether the implicit-str-concat should generate a warning
+# on implicit string concatenation in sequences defined over several lines.
+check-str-concat-over-line-jumps=no
+
+
+[TYPECHECK]
+
+# List of decorators that produce context managers, such as
+# contextlib.contextmanager. Add to this list to register other decorators that
+# produce valid context managers.
+contextmanager-decorators=contextlib.contextmanager
+
+# List of members which are set dynamically and missed by pylint inference
+# system, and so shouldn't trigger E1101 when accessed. Python regular
+# expressions are accepted.
+generated-members=firebase_admin.firestore.*,firestore.*
+
+# Tells whether to warn about missing members when the owner of the attribute
+# is inferred to be None.
+ignore-none=yes
+
+# This flag controls whether pylint should warn about no-member and similar
+# checks whenever an opaque object is returned when inferring. The inference
+# can return multiple potential results while evaluating a Python object, but
+# some branches might not be evaluated, which results in partial inference. In
+# that case, it might be useful to still emit no-member and other checks for
+# the rest of the inferred objects.
+ignore-on-opaque-inference=yes
+
+# List of symbolic message names to ignore for Mixin members.
+ignored-checks-for-mixins=no-member,
+                          not-async-context-manager,
+                          not-context-manager,
+                          attribute-defined-outside-init
+
+# List of class names for which member attributes should not be checked (useful
+# for classes with dynamically set attributes). This supports the use of
+# qualified names.
+ignored-classes=optparse.Values,thread._local,_thread._local,argparse.Namespace
+
+# Show a hint with possible names when a member name was not found. The aspect
+# of finding the hint is based on edit distance.
+missing-member-hint=yes
+
+# The minimum edit distance a name should have in order to be considered a
+# similar match for a missing member name.
+missing-member-hint-distance=1
+
+# The total number of similar names that should be taken in consideration when
+# showing a hint for a missing member.
+missing-member-max-choices=1
+
+# Regex pattern to define which classes are considered mixins.
+mixin-class-rgx=.*[Mm]ixin
+
+# List of decorators that change the signature of a decorated function.
+signature-mutators=
+
+
+[VARIABLES]
+
+# List of additional names supposed to be defined in builtins. Remember that
+# you should avoid defining new builtins when possible.
+additional-builtins=
+
+# Tells whether unused global variables should be treated as a violation.
+allow-global-unused-variables=yes
+
+# List of names allowed to shadow builtins
+allowed-redefined-builtins=
+
+# List of strings which can identify a callback function by name. A callback
+# name must start or end with one of those strings.
+callbacks=cb_,
+          _cb
+
+# A regular expression matching the name of dummy variables (i.e. expected to
+# not be used).
+dummy-variables-rgx=_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_
+
+# Argument names that match this expression will be ignored.
+ignored-argument-names=_.*|^ignored_|^unused_
+
+# Tells whether we should check for unused import in __init__ files.
+init-import=no
+
+# List of qualified module names which can have objects that can redefine
+# builtins.
+redefining-builtins-modules=six.moves,past.builtins,future.builtins,builtins,io

.vscode/launch.json

@@ -0,0 +1,47 @@
+{
+    // Use IntelliSense to learn about possible attributes.
+    // Hover to view descriptions of existing attributes.
+    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "name": "debug go",
+            "type": "go",
+            "request": "launch",
+            "mode": "exec",
+            "cwd": "${workspaceFolder}",
+            "program": "${workspaceFolder}/agents/bin/worker",
+            "env": {
+                "LD_LIBRARY_PATH": "${workspaceFolder}/agents/ten_packages/system/ten_runtime_go/lib:${workspaceFolder}/agents/ten_packages/system/agora_rtc_sdk/lib:${workspaceFolder}/agents/ten_packages/system/azure_speech_sdk/lib",
+                "TEN_APP_BASE_DIR": "${workspaceFolder}/agents"
+            }
+        },
+        {
+            "name": "debug python",
+            "type": "debugpy",
+            "request": "attach",
+            "connect": {
+                "host": "localhost",
+                "port": 5678
+            },
+            "preLaunchTask": "start app"
+        },
+        {
+            "name": "debug cpp",
+            "type": "cppdbg",
+            "request": "launch",
+            "program": "${workspaceFolder}/agents/bin/worker",
+            "cwd": "${workspaceFolder}",
+            "environment": [
+                {
+                    "name": "LD_LIBRARY_PATH",
+                    "value": "${workspaceFolder}/agents/ten_packages/system/agora_rtc_sdk/lib:${workspaceFolder}/agents/ten_packages/system/azure_speech_sdk/lib"
+                },
+                {
+                    "name": "CGO_LDFLAGS",
+                    "value": "-L${workspaceFolder}/agents/ten_packages/system/ten_runtime_go/lib -lten_runtime_go -Wl,-rpath,@loader_path/lib -Wl,-rpath,@loader_path/../lib"
+                }
+            ]
+        }
+    ]
+}

.vscode/settings.json

@@ -0,0 +1,13 @@
+{
+  "C_Cpp.intelliSenseEngine": "disabled",
+  "editor.formatOnSave": true,
+  "editor.defaultFormatter": null,
+  "[python]": {
+    "editor.defaultFormatter": "ms-python.black-formatter"
+  },
+  "git.ignoreLimitWarning": true,
+  "pylint.ignorePatterns": [
+    "*/ten_runtime_python/**/*",
+    "/usr/lib/**/*"
+  ],
+}

.vscode/tasks.json

@@ -0,0 +1,22 @@
+{
+	"version": "2.0.0",
+	"tasks": [
+		{
+			"type": "shell",
+			"label": "build",
+			"command": "make build",
+			"args": [],
+			"group": {
+				"kind": "build",
+				"isDefault": true
+			}
+		},
+		{
+			"label": "start app",
+			"type": "shell",
+			"command": "export TEN_ENABLE_PYTHON_DEBUG=true; export TEN_PYTHON_DEBUG_PORT=5678; ./agents/bin/start",
+			"group": "none",
+			"isBackground": true
+		},
+	]
+}

Dockerfile

@@ -22,25 +22,22 @@ RUN apt-get clean && apt-get update && apt-get install -y --no-install-recommend
     ca-certificates \
     && apt-get clean && rm -rf /var/lib/apt/lists/* && rm -rf /tmp/*
 
-# Создаем пользователя для запуска приложения с правильными правами
-RUN useradd -m -s /bin/bash tenuser
-
-# Установка Go 1.21 с правильными правами доступа
+# Установка Go 1.21
 RUN wget https://golang.org/dl/go1.21.0.linux-amd64.tar.gz && \
     tar -C /usr/local -xzf go1.21.0.linux-amd64.tar.gz && \
-    rm go1.21.0.linux-amd64.tar.gz && \
-    mkdir -p /home/tenuser/go && \
-    chown -R tenuser:tenuser /home/tenuser/go
-
+    rm go1.21.0.linux-amd64.tar.gz
 ENV PATH=$PATH:/usr/local/go/bin
-ENV GOPATH=/home/tenuser/go
+ENV GOPATH=/go
 ENV PATH=$PATH:$GOPATH/bin
 
-# Установка Node.js 20.x и pnpm для Playground
+# Установка Node.js 20.x (вместо 18.x) и pnpm для Playground
 RUN curl -fsSL https://deb.nodesource.com/setup_20.x | bash - && \
     apt-get install -y nodejs && \
     npm install -g pnpm
 
+# Создаем пользователя для запуска приложения с правильными правами
+RUN useradd -m -s /bin/bash tenuser
+
 WORKDIR /app
 
 # Клонирование репозитория TEN Agent (основная ветка)
@@ -73,7 +70,7 @@ RUN cd /app/server && \
 
 # Сборка Playground UI с правами пользователя
 WORKDIR /app/playground
-ENV PNPM_HOME="/home/tenuser/.pnpm-store"
+ENV PNPM_HOME="/app/.pnpm-store"
 ENV PATH="$PNPM_HOME:$PATH"
 RUN pnpm install --no-frozen-lockfile && \
     NEXT_PUBLIC_EDIT_GRAPH_MODE=false pnpm build

LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

Taskfile.yml

@@ -0,0 +1,120 @@
+version: '3'
+
+tasks:
+  clean:
+    desc: clean up
+    cmds:
+      - task: clean-agents
+      - task: clean-server
+
+  lint:
+    desc: lint-agent
+    env:
+      PYTHONPATH: "./agents/ten_packages/system/ten_runtime_python/lib:./agents/ten_packages/system/ten_runtime_python/interface:./agents/ten_packages/system/ten_ai_base/interface"
+    cmds:
+      - ./agents/scripts/pylint.sh
+
+  install-tools:
+    desc: install tools
+    cmds:
+      - pip install pylint
+
+  build:
+    desc: build
+    cmds:
+      - task: build-agent
+      - task: build-server
+
+  use:
+    desc: use agent, default 'agents/examples/default'
+    vars:
+      AGENT: '{{.AGENT| default "agents/examples/default"}}'
+    cmds:
+      - ln -sf {{.USER_WORKING_DIR}}/{{.AGENT}}/manifest.json ./agents/
+      - ln -sf {{.USER_WORKING_DIR}}/{{.AGENT}}/property.json ./agents/
+      - task: build
+
+  run-server:
+    desc: run backend http server
+    cmds:
+      - source .env && /app/server/bin/api
+
+  run-gd-server:
+    desc: run tman dev http server for graph designer
+    dir: ./agents
+    cmds:
+      - tman designer
+
+  run:
+    desc: run servers
+    deps:
+      - task: run-server
+      - task: run-gd-server
+
+  build-agent:
+    desc: build agent
+    dir: ./agents
+    internal: true
+    cmds:
+      - ./scripts/install_deps_and_build.sh linux x64 && mv bin/main bin/worker
+
+  build-server:
+    desc: build server
+    dir: ./server
+    cmds:
+      - go mod tidy && go mod download && go build -o bin/api main.go
+
+  clean-agents:
+    desc: clean up agents
+    dir: ./agents
+    internal: true
+    cmds:
+      - rm -rf manifest.json property.json manifest-lock.json bin/main bin/worker out .release ten_packages/system ten_packages/system/agora_rtc_sdk ten_packages/system/azure_speech_sdk ten_packages/system/nlohmann_json ten_packages/extension/agora_rtc ten_packages/extension/agora_rtm ten_packages/extension/agora_sess_ctrl ten_packages/extension/azure_tts ten_packages/addon_loader
+      - find . -type d -name .pytest_cache -exec rm -rf {} \; || true
+      - find . -type d -name __pycache__ -exec rm -rf {} \; || true
+      - find . -type d -name .ten -exec rm -rf {} \; || true
+      - find . -name .coverage -exec rm -f {} \; || true
+
+  clean-server:
+    desc: clean up server
+    dir: ./server
+    internal: true
+    cmds:
+      - rm -rf bin
+
+  test:
+    desc: run tests
+    cmds:
+      - task: test-agent-extensions
+      - task: test-server
+
+  test-server:
+    desc: test server
+    dir: ./server
+    internal: true
+    cmds:
+      - go test -v ./...
+
+  test-agent-extensions:
+    desc: run standalone testing of extensions
+    internal: true
+    env:
+      PYTHONPATH: "{{.USER_WORKING_DIR}}:{{.USER_WORKING_DIR}}/agents/ten_packages/system/ten_runtime_python/lib:{{.USER_WORKING_DIR}}/agents/ten_packages/system/ten_runtime_python/interface:{{.USER_WORKING_DIR}}/agents/ten_packages/system/ten_ai_base/interface"
+    vars:
+      EXTENSIONS:
+        sh: 'find agents/ten_packages/extension -type d -exec test -d "{}/tests" \; -print'
+    cmds:
+      - for: { var: EXTENSIONS }
+        task: test-extension
+        vars:
+          EXTENSION: '{{ .ITEM }}'
+
+  test-extension:
+    desc: run standalone testing of one single extension
+    vars:
+      EXTENSION: '{{.EXTENSION| default "agents/ten_packages/extension/elevenlabs_tts_python"}}'
+    env:
+      PYTHONPATH: "{{.USER_WORKING_DIR}}:{{.USER_WORKING_DIR}}/agents/ten_packages/system/ten_runtime_python/lib:{{.USER_WORKING_DIR}}/agents/ten_packages/system/ten_runtime_python/interface:{{.USER_WORKING_DIR}}/agents/ten_packages/system/ten_ai_base/interface"
+    dotenv: ['.env']
+    cmds:
+      - cd {{.EXTENSION}} && tman -y install --standalone && ./tests/bin/start {{ .CLI_ARGS }}

agents/.clang-format

@@ -0,0 +1,8 @@
+BasedOnStyle: Google
+SeparateDefinitionBlocks: Always
+ColumnLimit: 120
+BinPackArguments: false
+BinPackParameters: false
+AlignAfterOpenBracket: Align
+AllowAllArgumentsOnNextLine: false
+AllowAllParametersOfDeclarationOnNextLine: false

agents/.gitignore

@@ -0,0 +1,43 @@
+*.log
+ten_packages/extension_group/
+ten_packages/extension/agora_rtc
+ten_packages/extension/azure_tts
+ten_packages/extension/agora_sess_ctrl
+ten_packages/extension/agora_rtm
+ten_packages/extension/http_server_python
+ten_packages/system/agora_rtc_sdk
+ten_packages/system/azure_speech_sdk
+ten_packages/system/nlohmann_json
+ten_packages/system/ten_runtime*
+ten_packages/system
+ten_packages/addon_loader
+.ten
+agoradns.dat
+agorareport.dat
+agorartmreport.dat
+agora_cache.db
+bin/man
+bin/worker
+/BUILD.gn
+.cache/
+/compile_commands.json
+core
+crash_context_v1
+.deps/
+.DS_Store
+/.gn
+/.gnfiles
+include/
+interface/
+lib/
+/out/
+*.pcm
+.release
+session_control.conf.agora
+xdump_config
+.vscode
+*.pyc
+*.pyc.*
+/manifest.json
+/manifest-lock.json
+/property.json

agents/bin/start

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+set -e
+
+cd "$(dirname "${BASH_SOURCE[0]}")/.."
+
+#export TEN_ENABLE_PYTHON_DEBUG=true
+#export TEN_PYTHON_DEBUG_PORT=5678
+export PYTHONPATH=$(pwd)/ten_packages/system/ten_ai_base/interface:$PYTHONPATH
+export LD_LIBRARY_PATH=$(pwd)/ten_packages/system/agora_rtc_sdk/lib:$(pwd)/ten_packages/extension/agora_rtm/lib:$(pwd)/ten_packages/system/azure_speech_sdk/lib
+
+exec bin/worker "$@"

agents/examples/default/manifest.json

@@ -0,0 +1,152 @@
+{
+  "type": "app",
+  "name": "agent_demo",
+  "version": "0.8.0",
+  "dependencies": [
+    {
+      "type": "system",
+      "name": "ten_runtime_go",
+      "version": "0.8"
+    },
+    {
+      "type": "extension",
+      "name": "agora_rtc",
+      "version": "=0.12.0"
+    },
+    {
+      "type": "extension",
+      "name": "agora_sess_ctrl",
+      "version": "=0.4.4"
+    },
+    {
+      "type": "system",
+      "name": "azure_speech_sdk",
+      "version": "1.38.0"
+    },
+    {
+      "type": "system",
+      "name": "ten_ai_base",
+      "version": "0.4.1"
+    },
+    {
+      "type": "extension",
+      "name": "azure_tts",
+      "version": "=0.8.1"
+    },
+    {
+      "type": "extension",
+      "name": "openai_v2v_python",
+      "version": "=0.1.0"
+    },
+    {
+      "type": "extension",
+      "name": "message_collector",
+      "version": "=0.1.0"
+    },
+    {
+      "type": "extension",
+      "name": "bingsearch_tool_python",
+      "version": "=0.1.0"
+    },
+    {
+      "type": "extension",
+      "name": "openai_chatgpt_python",
+      "version": "=0.1.0"
+    },
+    {
+      "type": "extension",
+      "name": "fish_audio_tts",
+      "version": "=0.1.0"
+    },
+    {
+      "type": "extension",
+      "name": "interrupt_detector_python",
+      "version": "=0.1.0"
+    },
+    {
+      "type": "extension",
+      "name": "weatherapi_tool_python",
+      "version": "=0.1.0"
+    },
+    {
+      "type": "extension",
+      "name": "deepgram_asr_python",
+      "version": "=0.1.0"
+    },
+    {
+      "type": "extension",
+      "name": "vision_tool_python",
+      "version": "=0.1.0"
+    },
+    {
+      "type": "extension",
+      "name": "vision_analyze_tool_python",
+      "version": "=0.1.0"
+    },
+    {
+      "type": "extension",
+      "name": "transcribe_asr_python",
+      "version": "=0.1.0"
+    },
+    {
+      "type": "extension",
+      "name": "gemini_llm_python",
+      "version": "=0.1.0"
+    },
+    {
+      "type": "extension",
+      "name": "bedrock_llm_python",
+      "version": "=0.1.0"
+    },
+    {
+      "type": "extension",
+      "name": "polly_tts",
+      "version": "=0.1.0"
+    },
+    {
+      "type": "extension",
+      "name": "minimax_tts_python",
+      "version": "=0.1.0"
+    },
+    {
+      "type": "extension",
+      "name": "minimax_v2v_python",
+      "version": "=0.1.0"
+    },
+    {
+      "type": "extension",
+      "name": "cosy_tts_python",
+      "version": "=0.1.0"
+    },
+    {
+      "type": "extension",
+      "name": "elevenlabs_tts_python",
+      "version": "=0.1.0"
+    },
+    {
+      "type": "extension",
+      "name": "dify_python",
+      "version": "=0.1.0"
+    },
+    {
+      "type": "extension",
+      "name": "gemini_v2v_python",
+      "version": "=0.1.0"
+    },
+    {
+      "type": "extension",
+      "name": "coze_python_async",
+      "version": "=0.1.0"
+    },
+    {
+      "type": "extension",
+      "name": "openai_image_generate_tool",
+      "version": "=0.1.0"
+    },
+    {
+      "type": "extension",
+      "name": "computer_tool_python",
+      "version": "=0.1.0"
+    }
+  ]
+}

agents/examples/default/property.json

@@ -0,0 +1,1331 @@
+{
+  "_ten": {
+    "predefined_graphs": [
+      {
+        "name": "voice_assistant",
+        "auto_start": true,
+        "nodes": [
+          {
+            "type": "extension",
+            "name": "agora_rtc",
+            "addon": "agora_rtc",
+            "extension_group": "default",
+            "property": {
+              "app_id": "${env:AGORA_APP_ID}",
+              "token": "<agora_token>",
+              "channel": "ten_agent_test",
+              "stream_id": 1234,
+              "remote_stream_id": "123",
+              "subscribe_audio": true,
+              "publish_audio": true,
+              "publish_data": true,
+              "enable_agora_asr": false,
+              "agora_asr_vendor_name": "microsoft",
+              "agora_asr_language": "ru-RU",
+              "agora_asr_vendor_key": "${env:AZURE_STT_KEY|}",
+              "agora_asr_vendor_region": "${env:AZURE_STT_REGION|}",
+              "agora_asr_session_control_file_path": "session_control.conf"
+            }
+          },
+          {
+            "type": "extension",
+            "name": "stt",
+            "addon": "deepgram_asr_python",
+            "extension_group": "stt",
+            "property": {
+              "api_key": "${env:DEEPGRAM_API_KEY}",
+              "language": "ru-RU",
+              "model": "nova-2",
+              "sample_rate": 16000
+            }
+          },
+          {
+            "type": "extension",
+            "name": "llm",
+            "addon": "openai_chatgpt_python",
+            "extension_group": "chatgpt",
+            "property": {
+              "api_key": "${env:OPENAI_API_KEY}",
+              "base_url": "https://nitrox-fastflowwrapper.hf.space/v1",
+              "frequency_penalty": 0.9,
+              "greeting": "Я подключилась, о чем сегодня?",
+              "max_memory_length": 150,
+              "max_tokens": 512,
+              "model": "${env:OPENAI_MODEL}",
+              "prompt": "Your role is Friend. You provide emotional support, actively listen and reflect on the user's feelings, validate their emotions, and ask open-ended questions to deepen the conversation in a safe and non-judgmental space. Engage informally and share personal experiences while respecting boundaries and encouraging the user. Maintain clarity and conciseness by replying in no more than five sentences per answer using friendly English language. Your responses must always include answers to the user's inquiries while offering insights, comfort, and encouragement without using any special symbols. Your language is Russian.",
+              "proxy_url": "${env:OPENAI_PROXY_URL}"
+            }
+          },
+          {
+            "type": "extension",
+            "name": "tts",
+            "addon": "fish_audio_tts",
+            "extension_group": "tts",
+            "property": {
+              "api_key": "${env:FISH_AUDIO_TTS_KEY}",
+              "base_url": "https://api.fish.audio",
+              "model_id": "d61694f4ee5042aba2ffe11a9635d97e",
+              "optimize_streaming_latency": true,
+              "request_timeout_seconds": 30
+            }
+          },
+          {
+            "type": "extension",
+            "name": "interrupt_detector",
+            "addon": "interrupt_detector_python",
+            "extension_group": "default",
+            "property": {}
+          },
+          {
+            "type": "extension",
+            "name": "message_collector",
+            "addon": "message_collector",
+            "extension_group": "transcriber",
+            "property": {}
+          }
+        ],
+        "connections": [
+          {
+            "extension": "agora_rtc",
+            "cmd": [
+              {
+                "name": "on_user_joined",
+                "dest": [
+                  {
+                    "extension": "llm"
+                  }
+                ]
+              },
+              {
+                "name": "on_user_left",
+                "dest": [
+                  {
+                    "extension": "llm"
+                  }
+                ]
+              },
+              {
+                "name": "on_connection_failure",
+                "dest": [
+                  {
+                    "extension": "llm"
+                  }
+                ]
+              }
+            ],
+            "audio_frame": [
+              {
+                "name": "pcm_frame",
+                "dest": [
+                  {
+                    "extension": "stt"
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "extension": "stt",
+            "data": [
+              {
+                "name": "text_data",
+                "dest": [
+                  {
+                    "extension": "interrupt_detector"
+                  },
+                  {
+                    "extension": "message_collector"
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "extension": "llm",
+            "cmd": [
+              {
+                "name": "flush",
+                "dest": [
+                  {
+                    "extension": "tts"
+                  }
+                ]
+              }
+            ],
+            "data": [
+              {
+                "name": "text_data",
+                "dest": [
+                  {
+                    "extension": "tts"
+                  },
+                  {
+                    "extension": "message_collector"
+                  }
+                ]
+              },
+              {
+                "name": "content_data",
+                "dest": [
+                  {
+                    "extension": "message_collector"
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "extension": "message_collector",
+            "data": [
+              {
+                "name": "data",
+                "dest": [
+                  {
+                    "extension": "agora_rtc"
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "extension": "tts",
+            "cmd": [
+              {
+                "name": "flush",
+                "dest": [
+                  {
+                    "extension": "agora_rtc"
+                  }
+                ]
+              }
+            ],
+            "audio_frame": [
+              {
+                "name": "pcm_frame",
+                "dest": [
+                  {
+                    "extension": "agora_rtc"
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "extension": "interrupt_detector",
+            "cmd": [
+              {
+                "name": "flush",
+                "dest": [
+                  {
+                    "extension": "llm"
+                  }
+                ]
+              }
+            ],
+            "data": [
+              {
+                "name": "text_data",
+                "dest": [
+                  {
+                    "extension": "llm"
+                  }
+                ]
+              }
+            ]
+          }
+        ]
+      },
+      {
+        "name": "voice_assistant_integrated_stt",
+        "auto_start": true,
+        "nodes": [
+          {
+            "type": "extension",
+            "name": "agora_rtc",
+            "addon": "agora_rtc",
+            "extension_group": "default",
+            "property": {
+              "app_id": "${env:AGORA_APP_ID}",
+              "token": "<agora_token>",
+              "channel": "ten_agent_test",
+              "stream_id": 1234,
+              "remote_stream_id": 123,
+              "subscribe_audio": true,
+              "publish_audio": true,
+              "publish_data": true,
+              "enable_agora_asr": true,
+              "agora_asr_vendor_name": "microsoft",
+              "agora_asr_language": "en-US",
+              "agora_asr_vendor_key": "${env:AZURE_STT_KEY|}",
+              "agora_asr_vendor_region": "${env:AZURE_STT_REGION|}",
+              "agora_asr_session_control_file_path": "session_control.conf"
+            }
+          },
+          {
+            "type": "extension",
+            "name": "llm",
+            "addon": "openai_chatgpt_python",
+            "extension_group": "chatgpt",
+            "property": {
+              "api_key": "${env:OPENAI_API_KEY}",
+              "base_url": "",
+              "frequency_penalty": 0.9,
+              "greeting": "TEN Agent connected. How can I help you today?",
+              "max_memory_length": 10,
+              "max_tokens": 512,
+              "model": "${env:OPENAI_MODEL}",
+              "prompt": "",
+              "proxy_url": "${env:OPENAI_PROXY_URL}"
+            }
+          },
+          {
+            "type": "extension",
+            "name": "tts",
+            "addon": "fish_audio_tts",
+            "extension_group": "tts",
+            "property": {
+              "api_key": "${env:FISH_AUDIO_TTS_KEY}",
+              "model_id": "d8639b5cc95548f5afbcfe22d3ba5ce5",
+              "optimize_streaming_latency": true,
+              "request_timeout_seconds": 30,
+              "base_url": "https://api.fish.audio"
+            }
+          },
+          {
+            "type": "extension",
+            "name": "interrupt_detector",
+            "addon": "interrupt_detector_python",
+            "extension_group": "default",
+            "property": {}
+          },
+          {
+            "type": "extension",
+            "name": "message_collector",
+            "addon": "message_collector",
+            "extension_group": "transcriber",
+            "property": {}
+          },
+          {
+            "type": "extension",
+            "name": "weatherapi_tool_python",
+            "addon": "weatherapi_tool_python",
+            "extension_group": "default",
+            "property": {
+              "api_key": "${env:WEATHERAPI_API_KEY|}"
+            }
+          }
+        ],
+        "connections": [
+          {
+            "extension": "agora_rtc",
+            "cmd": [
+              {
+                "name": "on_user_joined",
+                "dest": [
+                  {
+                    "extension": "llm"
+                  }
+                ]
+              },
+              {
+                "name": "on_user_left",
+                "dest": [
+                  {
+                    "extension": "llm"
+                  }
+                ]
+              },
+              {
+                "name": "on_connection_failure",
+                "dest": [
+                  {
+                    "extension": "llm"
+                  }
+                ]
+              }
+            ],
+            "data": [
+              {
+                "name": "text_data",
+                "dest": [
+                  {
+                    "extension": "interrupt_detector"
+                  },
+                  {
+                    "extension": "message_collector"
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "extension": "llm",
+            "cmd": [
+              {
+                "name": "flush",
+                "dest": [
+                  {
+                    "extension": "tts"
+                  }
+                ]
+              },
+              {
+                "name": "tool_call",
+                "dest": [
+                  {
+                    "extension": "weatherapi_tool_python"
+                  }
+                ]
+              }
+            ],
+            "data": [
+              {
+                "name": "text_data",
+                "dest": [
+                  {
+                    "extension": "tts"
+                  },
+                  {
+                    "extension": "message_collector"
+                  }
+                ]
+              },
+              {
+                "name": "content_data",
+                "dest": [
+                  {
+                    "extension": "message_collector"
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "extension": "message_collector",
+            "data": [
+              {
+                "name": "data",
+                "dest": [
+                  {
+                    "extension": "agora_rtc"
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "extension": "tts",
+            "cmd": [
+              {
+                "name": "flush",
+                "dest": [
+                  {
+                    "extension": "agora_rtc"
+                  }
+                ]
+              }
+            ],
+            "audio_frame": [
+              {
+                "name": "pcm_frame",
+                "dest": [
+                  {
+                    "extension": "agora_rtc"
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "extension": "interrupt_detector",
+            "cmd": [
+              {
+                "name": "flush",
+                "dest": [
+                  {
+                    "extension": "llm"
+                  }
+                ]
+              }
+            ],
+            "data": [
+              {
+                "name": "text_data",
+                "dest": [
+                  {
+                    "extension": "llm"
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "extension": "weatherapi_tool_python",
+            "cmd": [
+              {
+                "name": "tool_register",
+                "dest": [
+                  {
+                    "extension": "llm"
+                  }
+                ]
+              }
+            ]
+          }
+        ]
+      },
+      {
+        "name": "voice_assistant_realtime",
+        "auto_start": true,
+        "nodes": [
+          {
+            "type": "extension",
+            "name": "agora_rtc",
+            "addon": "agora_rtc",
+            "extension_group": "rtc",
+            "property": {
+              "app_id": "${env:AGORA_APP_ID}",
+              "token": "",
+              "channel": "ten_agent_test",
+              "stream_id": 1234,
+              "remote_stream_id": 123,
+              "subscribe_audio": true,
+              "publish_audio": true,
+              "publish_data": true,
+              "subscribe_audio_sample_rate": 24000
+            }
+          },
+          {
+            "type": "extension",
+            "name": "v2v",
+            "addon": "openai_v2v_python",
+            "extension_group": "llm",
+            "property": {
+              "api_key": "${env:OPENAI_REALTIME_API_KEY}",
+              "enable_storage": false,
+              "history": 10,
+              "language": "en-US",
+              "max_tokens": 2048,
+              "model": "gpt-4o-realtime-preview",
+              "server_vad": true,
+              "temperature": 0.9,
+              "voice": "alloy"
+            }
+          },
+          {
+            "type": "extension",
+            "name": "message_collector",
+            "addon": "message_collector",
+            "extension_group": "transcriber",
+            "property": {}
+          },
+          {
+            "type": "extension",
+            "name": "weatherapi_tool_python",
+            "addon": "weatherapi_tool_python",
+            "extension_group": "default",
+            "property": {
+              "api_key": "${env:WEATHERAPI_API_KEY|}"
+            }
+          }
+        ],
+        "connections": [
+          {
+            "extension": "agora_rtc",
+            "cmd": [
+              {
+                "name": "on_user_joined",
+                "dest": [
+                  {
+                    "extension": "v2v"
+                  }
+                ]
+              },
+              {
+                "name": "on_user_left",
+                "dest": [
+                  {
+                    "extension": "v2v"
+                  }
+                ]
+              },
+              {
+                "name": "on_connection_failure",
+                "dest": [
+                  {
+                    "extension": "v2v"
+                  }
+                ]
+              }
+            ],
+            "audio_frame": [
+              {
+                "name": "pcm_frame",
+                "dest": [
+                  {
+                    "extension": "v2v"
+                  }
+                ]
+              }
+            ],
+            "video_frame": [
+              {
+                "name": "video_frame",
+                "dest": [
+                  {
+                    "extension": "v2v"
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "extension": "v2v",
+            "cmd": [
+              {
+                "name": "flush",
+                "dest": [
+                  {
+                    "extension": "agora_rtc"
+                  }
+                ]
+              },
+              {
+                "name": "tool_call",
+                "dest": [
+                  {
+                    "extension": "weatherapi_tool_python"
+                  }
+                ]
+              }
+            ],
+            "data": [
+              {
+                "name": "text_data",
+                "dest": [
+                  {
+                    "extension": "message_collector"
+                  }
+                ]
+              }
+            ],
+            "audio_frame": [
+              {
+                "name": "pcm_frame",
+                "dest": [
+                  {
+                    "extension": "agora_rtc"
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "extension": "message_collector",
+            "data": [
+              {
+                "name": "data",
+                "dest": [
+                  {
+                    "extension": "agora_rtc"
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "extension": "weatherapi_tool_python",
+            "cmd": [
+              {
+                "name": "tool_register",
+                "dest": [
+                  {
+                    "extension": "v2v"
+                  }
+                ]
+              }
+            ]
+          }
+        ]
+      },
+      {
+        "name": "story_teller",
+        "auto_start": true,
+        "nodes": [
+          {
+            "type": "extension",
+            "name": "agora_rtc",
+            "addon": "agora_rtc",
+            "extension_group": "default",
+            "property": {
+              "app_id": "${env:AGORA_APP_ID}",
+              "token": "<agora_token>",
+              "channel": "ten_agent_test",
+              "stream_id": 1234,
+              "remote_stream_id": 123,
+              "subscribe_audio": true,
+              "publish_audio": true,
+              "publish_data": true,
+              "enable_agora_asr": false
+            }
+          },
+          {
+            "type": "extension",
+            "name": "stt",
+            "addon": "deepgram_asr_python",
+            "extension_group": "stt",
+            "property": {
+              "api_key": "${env:DEEPGRAM_API_KEY}",
+              "language": "en-US",
+              "model": "nova-2",
+              "sample_rate": 16000
+            }
+          },
+          {
+            "type": "extension",
+            "name": "llm",
+            "addon": "openai_chatgpt_python",
+            "extension_group": "chatgpt",
+            "property": {
+              "api_key": "${env:OPENAI_API_KEY}",
+              "base_url": "",
+              "frequency_penalty": 0.9,
+              "greeting": "TEN Agent connected. How can I help you today?",
+              "max_memory_length": 10,
+              "max_tokens": 512,
+              "model": "${env:OPENAI_MODEL}",
+              "prompt": "You are an ai agent bot producing child picture books. Each response should be short and no more than 50 words as it's for child. \nFor every response relevant to the story-telling, you will use the 'image_generate' tool to create an image based on the description or key moment in that part of the story. \n The story should be set in a fantasy world. Try asking questions relevant to the story to decide how the story should proceed. Every response should include rich, vivid descriptions that will guide the 'image_generate' tool to produce an image that aligns with the scene or mood.\n Whether it’s the setting, a character’s expression, or a dramatic moment, the paragraph should give enough detail for a meaningful visual representation.",
+              "proxy_url": "${env:OPENAI_PROXY_URL}"
+            }
+          },
+          {
+            "type": "extension",
+            "name": "tts",
+            "addon": "fish_audio_tts",
+            "extension_group": "tts",
+            "property": {
+              "api_key": "${env:FISH_AUDIO_TTS_KEY}",
+              "model_id": "d8639b5cc95548f5afbcfe22d3ba5ce5",
+              "optimize_streaming_latency": true,
+              "request_timeout_seconds": 30,
+              "base_url": "https://api.fish.audio"
+            }
+          },
+          {
+            "type": "extension",
+            "name": "interrupt_detector",
+            "addon": "interrupt_detector_python",
+            "extension_group": "default",
+            "property": {}
+          },
+          {
+            "type": "extension",
+            "name": "message_collector",
+            "addon": "message_collector",
+            "extension_group": "transcriber",
+            "property": {}
+          },
+          {
+            "type": "extension",
+            "name": "openai_image_generate_tool",
+            "addon": "openai_image_generate_tool",
+            "extension_group": "default",
+            "property": {
+              "api_key": "${env:OPENAI_API_KEY}"
+            }
+          }
+        ],
+        "connections": [
+          {
+            "extension": "agora_rtc",
+            "cmd": [
+              {
+                "name": "on_user_joined",
+                "dest": [
+                  {
+                    "extension": "llm"
+                  }
+                ]
+              },
+              {
+                "name": "on_user_left",
+                "dest": [
+                  {
+                    "extension": "llm"
+                  }
+                ]
+              },
+              {
+                "name": "on_connection_failure",
+                "dest": [
+                  {
+                    "extension": "llm"
+                  }
+                ]
+              }
+            ],
+            "audio_frame": [
+              {
+                "name": "pcm_frame",
+                "dest": [
+                  {
+                    "extension": "stt"
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "extension": "stt",
+            "data": [
+              {
+                "name": "text_data",
+                "dest": [
+                  {
+                    "extension": "interrupt_detector"
+                  },
+                  {
+                    "extension": "message_collector"
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "extension": "llm",
+            "cmd": [
+              {
+                "name": "flush",
+                "dest": [
+                  {
+                    "extension": "tts"
+                  }
+                ]
+              },
+              {
+                "name": "tool_call",
+                "dest": [
+                  {
+                    "extension": "openai_image_generate_tool"
+                  }
+                ]
+              }
+            ],
+            "data": [
+              {
+                "name": "text_data",
+                "dest": [
+                  {
+                    "extension": "tts"
+                  },
+                  {
+                    "extension": "message_collector"
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "extension": "message_collector",
+            "data": [
+              {
+                "name": "data",
+                "dest": [
+                  {
+                    "extension": "agora_rtc"
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "extension": "tts",
+            "cmd": [
+              {
+                "name": "flush",
+                "dest": [
+                  {
+                    "extension": "agora_rtc"
+                  }
+                ]
+              }
+            ],
+            "audio_frame": [
+              {
+                "name": "pcm_frame",
+                "dest": [
+                  {
+                    "extension": "agora_rtc"
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "extension": "interrupt_detector",
+            "cmd": [
+              {
+                "name": "flush",
+                "dest": [
+                  {
+                    "extension": "llm"
+                  }
+                ]
+              }
+            ],
+            "data": [
+              {
+                "name": "text_data",
+                "dest": [
+                  {
+                    "extension": "llm"
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "extension": "openai_image_generate_tool",
+            "cmd": [
+              {
+                "name": "tool_register",
+                "dest": [
+                  {
+                    "extension": "llm"
+                  }
+                ]
+              }
+            ],
+            "data": [
+              {
+                "name": "content_data",
+                "dest": [
+                  {
+                    "extension": "message_collector"
+                  }
+                ]
+              }
+            ]
+          }
+        ]
+      },
+      {
+        "name": "story_teller_stt_integrated",
+        "auto_start": true,
+        "nodes": [
+          {
+            "type": "extension",
+            "name": "agora_rtc",
+            "addon": "agora_rtc",
+            "extension_group": "default",
+            "property": {
+              "app_id": "${env:AGORA_APP_ID}",
+              "token": "<agora_token>",
+              "channel": "ten_agent_test",
+              "stream_id": 1234,
+              "remote_stream_id": 123,
+              "subscribe_audio": true,
+              "publish_audio": true,
+              "publish_data": true,
+              "enable_agora_asr": true,
+              "agora_asr_vendor_name": "microsoft",
+              "agora_asr_language": "en-US",
+              "agora_asr_vendor_key": "${env:AZURE_STT_KEY|}",
+              "agora_asr_vendor_region": "${env:AZURE_STT_REGION|}",
+              "agora_asr_session_control_file_path": "session_control.conf"
+            }
+          },
+          {
+            "type": "extension",
+            "name": "llm",
+            "addon": "openai_chatgpt_python",
+            "extension_group": "chatgpt",
+            "property": {
+              "api_key": "${env:OPENAI_API_KEY}",
+              "base_url": "",
+              "frequency_penalty": 0.9,
+              "greeting": "TEN Agent connected. How can I help you today?",
+              "max_memory_length": 10,
+              "max_tokens": 512,
+              "model": "${env:OPENAI_MODEL}",
+              "prompt": "You are an ai agent bot producing child picture books. Each response should be short and no more than 50 words as it's for child. \nFor every response relevant to the story-telling, you will use the 'image_generate' tool to create an image based on the description or key moment in that part of the story. \n The story should be set in a fantasy world. Try asking questions relevant to the story to decide how the story should proceed. Every response should include rich, vivid descriptions that will guide the 'image_generate' tool to produce an image that aligns with the scene or mood.\n Whether it’s the setting, a character’s expression, or a dramatic moment, the paragraph should give enough detail for a meaningful visual representation.",
+              "proxy_url": "${env:OPENAI_PROXY_URL}"
+            }
+          },
+          {
+            "type": "extension",
+            "name": "tts",
+            "addon": "fish_audio_tts",
+            "extension_group": "tts",
+            "property": {
+              "api_key": "${env:FISH_AUDIO_TTS_KEY}",
+              "model_id": "d8639b5cc95548f5afbcfe22d3ba5ce5",
+              "optimize_streaming_latency": true,
+              "request_timeout_seconds": 30,
+              "base_url": "https://api.fish.audio"
+            }
+          },
+          {
+            "type": "extension",
+            "name": "interrupt_detector",
+            "addon": "interrupt_detector_python",
+            "extension_group": "default",
+            "property": {}
+          },
+          {
+            "type": "extension",
+            "name": "message_collector",
+            "addon": "message_collector",
+            "extension_group": "transcriber",
+            "property": {}
+          },
+          {
+            "type": "extension",
+            "name": "openai_image_generate_tool",
+            "addon": "openai_image_generate_tool",
+            "extension_group": "default",
+            "property": {
+              "api_key": "${env:OPENAI_API_KEY}"
+            }
+          }
+        ],
+        "connections": [
+          {
+            "extension": "agora_rtc",
+            "cmd": [
+              {
+                "name": "on_user_joined",
+                "dest": [
+                  {
+                    "extension": "llm"
+                  }
+                ]
+              },
+              {
+                "name": "on_user_left",
+                "dest": [
+                  {
+                    "extension": "llm"
+                  }
+                ]
+              },
+              {
+                "name": "on_connection_failure",
+                "dest": [
+                  {
+                    "extension": "llm"
+                  }
+                ]
+              }
+            ],
+            "data": [
+              {
+                "name": "text_data",
+                "dest": [
+                  {
+                    "extension": "interrupt_detector"
+                  },
+                  {
+                    "extension": "message_collector"
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "extension": "llm",
+            "cmd": [
+              {
+                "name": "flush",
+                "dest": [
+                  {
+                    "extension": "tts"
+                  }
+                ]
+              },
+              {
+                "name": "tool_call",
+                "dest": [
+                  {
+                    "extension": "openai_image_generate_tool"
+                  }
+                ]
+              }
+            ],
+            "data": [
+              {
+                "name": "text_data",
+                "dest": [
+                  {
+                    "extension": "tts"
+                  },
+                  {
+                    "extension": "message_collector"
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "extension": "message_collector",
+            "data": [
+              {
+                "name": "data",
+                "dest": [
+                  {
+                    "extension": "agora_rtc"
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "extension": "tts",
+            "cmd": [
+              {
+                "name": "flush",
+                "dest": [
+                  {
+                    "extension": "agora_rtc"
+                  }
+                ]
+              }
+            ],
+            "audio_frame": [
+              {
+                "name": "pcm_frame",
+                "dest": [
+                  {
+                    "extension": "agora_rtc"
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "extension": "interrupt_detector",
+            "cmd": [
+              {
+                "name": "flush",
+                "dest": [
+                  {
+                    "extension": "llm"
+                  }
+                ]
+              }
+            ],
+            "data": [
+              {
+                "name": "text_data",
+                "dest": [
+                  {
+                    "extension": "llm"
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "extension": "openai_image_generate_tool",
+            "cmd": [
+              {
+                "name": "tool_register",
+                "dest": [
+                  {
+                    "extension": "llm"
+                  }
+                ]
+              }
+            ],
+            "data": [
+              {
+                "name": "content_data",
+                "dest": [
+                  {
+                    "extension": "message_collector"
+                  }
+                ]
+              }
+            ]
+          }
+        ]
+      },
+      {
+        "name": "story_teller_realtime",
+        "auto_start": true,
+        "nodes": [
+          {
+            "type": "extension",
+            "name": "agora_rtc",
+            "addon": "agora_rtc",
+            "extension_group": "rtc",
+            "property": {
+              "app_id": "${env:AGORA_APP_ID}",
+              "token": "",
+              "channel": "ten_agent_test",
+              "stream_id": 1234,
+              "remote_stream_id": 123,
+              "subscribe_audio": true,
+              "publish_audio": true,
+              "publish_data": true,
+              "subscribe_audio_sample_rate": 24000
+            }
+          },
+          {
+            "type": "extension",
+            "name": "v2v",
+            "addon": "openai_v2v_python",
+            "extension_group": "llm",
+            "property": {
+              "api_key": "${env:OPENAI_REALTIME_API_KEY}",
+              "temperature": 0.9,
+              "model": "gpt-4o-realtime-preview-2024-12-17",
+              "max_tokens": 2048,
+              "voice": "alloy",
+              "language": "en-US",
+              "server_vad": true,
+              "prompt": "You are an ai agent bot producing child picture books. Each response should be short and no more than 50 words as it's for child. \nFor every response relevant to the story-telling, you will use the 'image_generate' tool to create an image based on the description or key moment in that part of the story. \n The story should be set in a fantasy world. Try asking questions relevant to the story to decide how the story should proceed. Every response should include rich, vivid descriptions that will guide the 'image_generate' tool to produce an image that aligns with the scene or mood.\n Whether it’s the setting, a character’s expression, or a dramatic moment, the paragraph should give enough detail for a meaningful visual representation.",
+              "dump": false,
+              "max_history": 10
+            }
+          },
+          {
+            "type": "extension",
+            "name": "message_collector",
+            "addon": "message_collector",
+            "extension_group": "transcriber",
+            "property": {}
+          },
+          {
+            "type": "extension",
+            "name": "openai_image_generate_tool",
+            "addon": "openai_image_generate_tool",
+            "extension_group": "default",
+            "property": {
+              "api_key": "${env:OPENAI_API_KEY}"
+            }
+          }
+        ],
+        "connections": [
+          {
+            "extension": "agora_rtc",
+            "cmd": [
+              {
+                "name": "on_user_joined",
+                "dest": [
+                  {
+                    "extension": "v2v"
+                  }
+                ]
+              },
+              {
+                "name": "on_user_left",
+                "dest": [
+                  {
+                    "extension": "v2v"
+                  }
+                ]
+              },
+              {
+                "name": "on_connection_failure",
+                "dest": [
+                  {
+                    "extension": "v2v"
+                  }
+                ]
+              }
+            ],
+            "audio_frame": [
+              {
+                "name": "pcm_frame",
+                "dest": [
+                  {
+                    "extension": "v2v"
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "extension": "v2v",
+            "cmd": [
+              {
+                "name": "flush",
+                "dest": [
+                  {
+                    "extension": "agora_rtc"
+                  }
+                ]
+              },
+              {
+                "name": "tool_call",
+                "dest": [
+                  {
+                    "extension": "openai_image_generate_tool"
+                  }
+                ]
+              }
+            ],
+            "data": [
+              {
+                "name": "text_data",
+                "dest": [
+                  {
+                    "extension": "message_collector"
+                  }
+                ]
+              }
+            ],
+            "audio_frame": [
+              {
+                "name": "pcm_frame",
+                "dest": [
+                  {
+                    "extension": "agora_rtc"
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "extension": "message_collector",
+            "data": [
+              {
+                "name": "data",
+                "dest": [
+                  {
+                    "extension": "agora_rtc"
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "extension": "openai_image_generate_tool",
+            "cmd": [
+              {
+                "name": "tool_register",
+                "dest": [
+                  {
+                    "extension": "v2v"
+                  }
+                ]
+              }
+            ],
+            "data": [
+              {
+                "name": "content_data",
+                "dest": [
+                  {
+                    "extension": "message_collector"
+                  }
+                ]
+              }
+            ]
+          }
+        ]
+      }
+    ],
+    "log_level": 3
+  }
+}

agents/examples/demo/manifest.json

@@ -0,0 +1,97 @@
+{
+  "type": "app",
+  "name": "agent_demo",
+  "version": "0.8.0",
+  "dependencies": [
+    {
+      "type": "system",
+      "name": "ten_runtime_go",
+      "version": "0.8"
+    },
+    {
+      "type": "extension",
+      "name": "agora_rtc",
+      "version": "=0.12.0"
+    },
+    {
+      "type": "extension",
+      "name": "agora_sess_ctrl",
+      "version": "=0.4.4"
+    },
+    {
+      "type": "system",
+      "name": "azure_speech_sdk",
+      "version": "1.38.0"
+    },
+    {
+      "type": "system",
+      "name": "ten_ai_base",
+      "version": "0.4.1"
+    },
+    {
+      "type": "extension",
+      "name": "azure_tts",
+      "version": "=0.8.1"
+    },
+    {
+      "type": "extension",
+      "name": "dify_python",
+      "version": "=0.1.0"
+    },
+    {
+      "type": "extension",
+      "name": "gemini_v2v_python",
+      "version": "=0.1.0"
+    },
+    {
+      "type": "extension",
+      "name": "openai_chatgpt_python",
+      "version": "=0.1.0"
+    },
+    {
+      "type": "extension",
+      "name": "bingsearch_tool_python",
+      "version": "=0.1.0"
+    },
+    {
+      "type": "extension",
+      "name": "vision_tool_python",
+      "version": "=0.1.0"
+    },
+    {
+      "type": "extension",
+      "name": "weatherapi_tool_python",
+      "version": "=0.1.0"
+    },
+    {
+      "type": "extension",
+      "name": "interrupt_detector_python",
+      "version": "=0.1.0"
+    },
+    {
+      "type": "extension",
+      "name": "openai_v2v_python",
+      "version": "=0.1.0"
+    },
+    {
+      "type": "extension",
+      "name": "message_collector",
+      "version": "=0.1.0"
+    },
+    {
+      "type": "extension",
+      "name": "coze_python_async",
+      "version": "=0.1.0"
+    },
+    {
+      "type": "extension",
+      "name": "fish_audio_tts",
+      "version": "=0.1.0"
+    },
+    {
+      "type": "extension",
+      "name": "openai_image_generate_tool",
+      "version": "=0.1.0"
+    }
+  ]
+}

agents/examples/demo/property.json

@@ -0,0 +1,2322 @@
+{
+  "_ten": {
+    "predefined_graphs": [
+      {
+        "name": "qwq_32b",
+        "auto_start": true,
+        "nodes": [
+          {
+            "type": "extension",
+            "name": "agora_rtc",
+            "addon": "agora_rtc",
+            "extension_group": "default",
+            "property": {
+              "app_id": "${env:AGORA_APP_ID}",
+              "token": "<agora_token>",
+              "channel": "ten_agent_test",
+              "stream_id": 1234,
+              "remote_stream_id": 123,
+              "subscribe_audio": true,
+              "publish_audio": true,
+              "publish_data": true,
+              "enable_agora_asr": true,
+              "agora_asr_vendor_name": "microsoft",
+              "agora_asr_language": "en-US",
+              "agora_asr_vendor_key": "${env:AZURE_STT_KEY|}",
+              "agora_asr_vendor_region": "${env:AZURE_STT_REGION|}",
+              "agora_asr_session_control_file_path": "session_control.conf"
+            }
+          },
+          {
+            "type": "extension",
+            "name": "llm",
+            "addon": "openai_chatgpt_python",
+            "extension_group": "chatgpt",
+            "property": {
+              "api_key": "${env:QWEN_API_KEY}",
+              "base_url": "https://dashscope.aliyuncs.com/compatible-mode/v1",
+              "frequency_penalty": 0.9,
+              "greeting": "TEN Agent connected. How can I help you today?",
+              "max_memory_length": 10,
+              "max_tokens": 512,
+              "model": "qwq-plus",
+              "prompt": "",
+              "proxy_url": "${env:OPENAI_PROXY_URL}"
+            }
+          },
+          {
+            "type": "extension",
+            "name": "tts",
+            "addon": "azure_tts",
+            "extension_group": "tts",
+            "property": {
+              "azure_subscription_key": "${env:AZURE_TTS_KEY}",
+              "azure_subscription_region": "${env:AZURE_TTS_REGION}",
+              "azure_synthesis_voice_name": "en-US-AndrewMultilingualNeural"
+            }
+          },
+          {
+            "type": "extension",
+            "name": "interrupt_detector",
+            "addon": "interrupt_detector_python",
+            "extension_group": "default",
+            "property": {}
+          },
+          {
+            "type": "extension",
+            "name": "message_collector",
+            "addon": "message_collector",
+            "extension_group": "transcriber",
+            "property": {}
+          }
+        ],
+        "connections": [
+          {
+            "extension": "agora_rtc",
+            "cmd": [
+              {
+                "name": "on_user_joined",
+                "dest": [
+                  {
+                    "extension": "llm"
+                  }
+                ]
+              },
+              {
+                "name": "on_user_left",
+                "dest": [
+                  {
+                    "extension": "llm"
+                  }
+                ]
+              },
+              {
+                "name": "on_connection_failure",
+                "dest": [
+                  {
+                    "extension": "llm"
+                  }
+                ]
+              }
+            ],
+            "data": [
+              {
+                "name": "text_data",
+                "dest": [
+                  {
+                    "extension": "interrupt_detector"
+                  },
+                  {
+                    "extension": "message_collector"
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "extension": "llm",
+            "cmd": [
+              {
+                "name": "flush",
+                "dest": [
+                  {
+                    "extension": "tts"
+                  }
+                ]
+              }
+            ],
+            "data": [
+              {
+                "name": "text_data",
+                "dest": [
+                  {
+                    "extension": "tts"
+                  },
+                  {
+                    "extension": "message_collector"
+                  }
+                ]
+              },
+              {
+                "name": "content_data",
+                "dest": [
+                  {
+                    "extension": "message_collector"
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "extension": "message_collector",
+            "data": [
+              {
+                "name": "data",
+                "dest": [
+                  {
+                    "extension": "agora_rtc"
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "extension": "tts",
+            "cmd": [
+              {
+                "name": "flush",
+                "dest": [
+                  {
+                    "extension": "agora_rtc"
+                  }
+                ]
+              }
+            ],
+            "audio_frame": [
+              {
+                "name": "pcm_frame",
+                "dest": [
+                  {
+                    "extension": "agora_rtc"
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "extension": "interrupt_detector",
+            "cmd": [
+              {
+                "name": "flush",
+                "dest": [
+                  {
+                    "extension": "llm"
+                  }
+                ]
+              }
+            ],
+            "data": [
+              {
+                "name": "text_data",
+                "dest": [
+                  {
+                    "extension": "llm"
+                  }
+                ]
+              }
+            ]
+          }
+        ]
+      },
+      {
+        "name": "deepseek_r1",
+        "auto_start": true,
+        "nodes": [
+          {
+            "type": "extension",
+            "name": "agora_rtc",
+            "addon": "agora_rtc",
+            "extension_group": "default",
+            "property": {
+              "app_id": "${env:AGORA_APP_ID}",
+              "token": "<agora_token>",
+              "channel": "ten_agent_test",
+              "stream_id": 1234,
+              "remote_stream_id": 123,
+              "subscribe_audio": true,
+              "publish_audio": true,
+              "publish_data": true,
+              "enable_agora_asr": true,
+              "agora_asr_vendor_name": "microsoft",
+              "agora_asr_language": "en-US",
+              "agora_asr_vendor_key": "${env:AZURE_STT_KEY|}",
+              "agora_asr_vendor_region": "${env:AZURE_STT_REGION|}",
+              "agora_asr_session_control_file_path": "session_control.conf"
+            }
+          },
+          {
+            "type": "extension",
+            "name": "llm",
+            "addon": "openai_chatgpt_python",
+            "extension_group": "chatgpt",
+            "property": {
+              "api_key": "${env:DEEPSEEK_API_KEY}",
+              "base_url": "https://tenagentopenai.services.ai.azure.com/models",
+              "frequency_penalty": 0.9,
+              "greeting": "TEN Agent connected. How can I help you today?",
+              "max_memory_length": 10,
+              "max_tokens": 512,
+              "model": "DeepSeek-R1",
+              "prompt": "",
+              "proxy_url": "${env:OPENAI_PROXY_URL}"
+            }
+          },
+          {
+            "type": "extension",
+            "name": "tts",
+            "addon": "azure_tts",
+            "extension_group": "tts",
+            "property": {
+              "azure_subscription_key": "${env:AZURE_TTS_KEY}",
+              "azure_subscription_region": "${env:AZURE_TTS_REGION}",
+              "azure_synthesis_voice_name": "en-US-AndrewMultilingualNeural"
+            }
+          },
+          {
+            "type": "extension",
+            "name": "interrupt_detector",
+            "addon": "interrupt_detector_python",
+            "extension_group": "default",
+            "property": {}
+          },
+          {
+            "type": "extension",
+            "name": "message_collector",
+            "addon": "message_collector",
+            "extension_group": "transcriber",
+            "property": {}
+          }
+        ],
+        "connections": [
+          {
+            "extension": "agora_rtc",
+            "cmd": [
+              {
+                "name": "on_user_joined",
+                "dest": [
+                  {
+                    "extension": "llm"
+                  }
+                ]
+              },
+              {
+                "name": "on_user_left",
+                "dest": [
+                  {
+                    "extension": "llm"
+                  }
+                ]
+              },
+              {
+                "name": "on_connection_failure",
+                "dest": [
+                  {
+                    "extension": "llm"
+                  }
+                ]
+              }
+            ],
+            "data": [
+              {
+                "name": "text_data",
+                "dest": [
+                  {
+                    "extension": "interrupt_detector"
+                  },
+                  {
+                    "extension": "message_collector"
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "extension": "llm",
+            "cmd": [
+              {
+                "name": "flush",
+                "dest": [
+                  {
+                    "extension": "tts"
+                  }
+                ]
+              }
+            ],
+            "data": [
+              {
+                "name": "text_data",
+                "dest": [
+                  {
+                    "extension": "tts"
+                  },
+                  {
+                    "extension": "message_collector"
+                  }
+                ]
+              },
+              {
+                "name": "content_data",
+                "dest": [
+                  {
+                    "extension": "message_collector"
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "extension": "message_collector",
+            "data": [
+              {
+                "name": "data",
+                "dest": [
+                  {
+                    "extension": "agora_rtc"
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "extension": "tts",
+            "cmd": [
+              {
+                "name": "flush",
+                "dest": [
+                  {
+                    "extension": "agora_rtc"
+                  }
+                ]
+              }
+            ],
+            "audio_frame": [
+              {
+                "name": "pcm_frame",
+                "dest": [
+                  {
+                    "extension": "agora_rtc"
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "extension": "interrupt_detector",
+            "cmd": [
+              {
+                "name": "flush",
+                "dest": [
+                  {
+                    "extension": "llm"
+                  }
+                ]
+              }
+            ],
+            "data": [
+              {
+                "name": "text_data",
+                "dest": [
+                  {
+                    "extension": "llm"
+                  }
+                ]
+              }
+            ]
+          }
+        ]
+      },
+      {
+        "name": "voice_assistant_realtime",
+        "auto_start": true,
+        "nodes": [
+          {
+            "type": "extension",
+            "name": "agora_rtc",
+            "addon": "agora_rtc",
+            "extension_group": "rtc",
+            "property": {
+              "app_id": "${env:AGORA_APP_ID}",
+              "token": "",
+              "channel": "ten_agent_test",
+              "stream_id": 1234,
+              "remote_stream_id": 123,
+              "subscribe_audio": true,
+              "publish_audio": true,
+              "publish_data": true,
+              "subscribe_audio_sample_rate": 24000
+            }
+          },
+          {
+            "type": "extension",
+            "name": "v2v",
+            "addon": "openai_v2v_python",
+            "extension_group": "llm",
+            "property": {
+              "api_key": "${env:OPENAI_REALTIME_API_KEY}",
+              "temperature": 0.9,
+              "model": "gpt-4o-realtime-preview-2024-12-17",
+              "max_tokens": 2048,
+              "voice": "alloy",
+              "language": "en-US",
+              "server_vad": true,
+              "dump": true,
+              "max_history": 10
+            }
+          },
+          {
+            "type": "extension",
+            "name": "message_collector",
+            "addon": "message_collector",
+            "extension_group": "transcriber",
+            "property": {}
+          },
+          {
+            "type": "extension",
+            "name": "weatherapi_tool_python",
+            "addon": "weatherapi_tool_python",
+            "extension_group": "default",
+            "property": {
+              "api_key": "${env:WEATHERAPI_API_KEY|}"
+            }
+          }
+        ],
+        "connections": [
+          {
+            "extension": "agora_rtc",
+            "cmd": [
+              {
+                "name": "on_user_joined",
+                "dest": [
+                  {
+                    "extension": "v2v"
+                  }
+                ]
+              },
+              {
+                "name": "on_user_left",
+                "dest": [
+                  {
+                    "extension": "v2v"
+                  }
+                ]
+              },
+              {
+                "name": "on_connection_failure",
+                "dest": [
+                  {
+                    "extension": "v2v"
+                  }
+                ]
+              }
+            ],
+            "audio_frame": [
+              {
+                "name": "pcm_frame",
+                "dest": [
+                  {
+                    "extension": "v2v"
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "extension": "v2v",
+            "cmd": [
+              {
+                "name": "flush",
+                "dest": [
+                  {
+                    "extension": "agora_rtc"
+                  }
+                ]
+              },
+              {
+                "name": "tool_call",
+                "dest": [
+                  {
+                    "extension": "weatherapi_tool_python"
+                  }
+                ]
+              }
+            ],
+            "data": [
+              {
+                "name": "text_data",
+                "dest": [
+                  {
+                    "extension": "message_collector"
+                  }
+                ]
+              }
+            ],
+            "audio_frame": [
+              {
+                "name": "pcm_frame",
+                "dest": [
+                  {
+                    "extension": "agora_rtc"
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "extension": "message_collector",
+            "data": [
+              {
+                "name": "data",
+                "dest": [
+                  {
+                    "extension": "agora_rtc"
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "extension": "weatherapi_tool_python",
+            "cmd": [
+              {
+                "name": "tool_register",
+                "dest": [
+                  {
+                    "extension": "v2v"
+                  }
+                ]
+              }
+            ]
+          }
+        ]
+      },
+      {
+        "name": "va_openai_azure",
+        "auto_start": true,
+        "nodes": [
+          {
+            "type": "extension",
+            "name": "agora_rtc",
+            "addon": "agora_rtc",
+            "extension_group": "default",
+            "property": {
+              "app_id": "${env:AGORA_APP_ID}",
+              "token": "<agora_token>",
+              "channel": "ten_agent_test",
+              "stream_id": 1234,
+              "remote_stream_id": 123,
+              "subscribe_audio": true,
+              "publish_audio": true,
+              "publish_data": true,
+              "enable_agora_asr": true,
+              "agora_asr_vendor_name": "microsoft",
+              "agora_asr_language": "en-US",
+              "agora_asr_vendor_key": "${env:AZURE_STT_KEY|}",
+              "agora_asr_vendor_region": "${env:AZURE_STT_REGION|}",
+              "agora_asr_session_control_file_path": "session_control.conf",
+              "subscribe_video_pix_fmt": 4,
+              "subscribe_video": true
+            }
+          },
+          {
+            "type": "extension",
+            "name": "llm",
+            "addon": "openai_chatgpt_python",
+            "extension_group": "chatgpt",
+            "property": {
+              "api_key": "${env:OPENAI_API_KEY}",
+              "base_url": "",
+              "frequency_penalty": 0.9,
+              "greeting": "TEN Agent connected. How can I help you today?",
+              "max_memory_length": 10,
+              "max_tokens": 512,
+              "model": "${env:OPENAI_MODEL}",
+              "prompt": "",
+              "proxy_url": "${env:OPENAI_PROXY_URL}"
+            }
+          },
+          {
+            "type": "extension",
+            "name": "tts",
+            "addon": "azure_tts",
+            "extension_group": "tts",
+            "property": {
+              "azure_subscription_key": "${env:AZURE_TTS_KEY}",
+              "azure_subscription_region": "${env:AZURE_TTS_REGION}",
+              "azure_synthesis_voice_name": "en-US-AndrewMultilingualNeural"
+            }
+          },
+          {
+            "type": "extension",
+            "name": "interrupt_detector",
+            "addon": "interrupt_detector_python",
+            "extension_group": "default",
+            "property": {}
+          },
+          {
+            "type": "extension",
+            "name": "message_collector",
+            "addon": "message_collector",
+            "extension_group": "transcriber",
+            "property": {}
+          },
+          {
+            "type": "extension",
+            "name": "weatherapi_tool_python",
+            "addon": "weatherapi_tool_python",
+            "extension_group": "default",
+            "property": {
+              "api_key": "${env:WEATHERAPI_API_KEY|}"
+            }
+          },
+          {
+            "type": "extension",
+            "name": "vision_tool_python",
+            "addon": "vision_tool_python",
+            "extension_group": "default",
+            "property": {}
+          },
+          {
+            "type": "extension",
+            "name": "bingsearch_tool_python",
+            "addon": "bingsearch_tool_python",
+            "extension_group": "default",
+            "property": {
+              "api_key": "${env:BING_API_KEY|}"
+            }
+          }
+        ],
+        "connections": [
+          {
+            "extension": "agora_rtc",
+            "cmd": [
+              {
+                "name": "on_user_joined",
+                "dest": [
+                  {
+                    "extension": "llm"
+                  }
+                ]
+              },
+              {
+                "name": "on_user_left",
+                "dest": [
+                  {
+                    "extension": "llm"
+                  }
+                ]
+              },
+              {
+                "name": "on_connection_failure",
+                "dest": [
+                  {
+                    "extension": "llm"
+                  }
+                ]
+              }
+            ],
+            "data": [
+              {
+                "name": "text_data",
+                "dest": [
+                  {
+                    "extension": "interrupt_detector"
+                  },
+                  {
+                    "extension": "message_collector"
+                  }
+                ]
+              }
+            ],
+            "video_frame": [
+              {
+                "name": "video_frame",
+                "dest": [
+                  {
+                    "extension": "vision_tool_python"
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "extension": "llm",
+            "cmd": [
+              {
+                "name": "flush",
+                "dest": [
+                  {
+                    "extension": "tts"
+                  }
+                ]
+              },
+              {
+                "name": "tool_call",
+                "dest": [
+                  {
+                    "extension": "weatherapi_tool_python"
+                  },
+                  {
+                    "extension": "vision_tool_python"
+                  },
+                  {
+                    "extension": "bingsearch_tool_python"
+                  }
+                ]
+              }
+            ],
+            "data": [
+              {
+                "name": "text_data",
+                "dest": [
+                  {
+                    "extension": "tts"
+                  },
+                  {
+                    "extension": "message_collector"
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "extension": "message_collector",
+            "data": [
+              {
+                "name": "data",
+                "dest": [
+                  {
+                    "extension": "agora_rtc"
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "extension": "tts",
+            "cmd": [
+              {
+                "name": "flush",
+                "dest": [
+                  {
+                    "extension": "agora_rtc"
+                  }
+                ]
+              }
+            ],
+            "audio_frame": [
+              {
+                "name": "pcm_frame",
+                "dest": [
+                  {
+                    "extension": "agora_rtc"
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "extension": "interrupt_detector",
+            "cmd": [
+              {
+                "name": "flush",
+                "dest": [
+                  {
+                    "extension": "llm"
+                  }
+                ]
+              }
+            ],
+            "data": [
+              {
+                "name": "text_data",
+                "dest": [
+                  {
+                    "extension": "llm"
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "extension": "weatherapi_tool_python",
+            "cmd": [
+              {
+                "name": "tool_register",
+                "dest": [
+                  {
+                    "extension": "llm"
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "extension": "vision_tool_python",
+            "cmd": [
+              {
+                "name": "tool_register",
+                "dest": [
+                  {
+                    "extension": "llm"
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "extension": "bingsearch_tool_python",
+            "cmd": [
+              {
+                "name": "tool_register",
+                "dest": [
+                  {
+                    "extension": "llm"
+                  }
+                ]
+              }
+            ]
+          }
+        ]
+      },
+      {
+        "name": "va_openai_v2v",
+        "auto_start": true,
+        "nodes": [
+          {
+            "type": "extension",
+            "name": "agora_rtc",
+            "addon": "agora_rtc",
+            "extension_group": "rtc",
+            "property": {
+              "app_id": "${env:AGORA_APP_ID}",
+              "token": "",
+              "channel": "ten_agent_test",
+              "stream_id": 1234,
+              "remote_stream_id": 123,
+              "subscribe_audio": true,
+              "publish_audio": true,
+              "publish_data": true,
+              "subscribe_audio_sample_rate": 24000
+            }
+          },
+          {
+            "type": "extension",
+            "name": "v2v",
+            "addon": "openai_v2v_python",
+            "extension_group": "llm",
+            "property": {
+              "api_key": "${env:OPENAI_REALTIME_API_KEY}",
+              "temperature": 0.9,
+              "model": "gpt-4o-realtime-preview-2024-12-17",
+              "max_tokens": 2048,
+              "voice": "alloy",
+              "language": "en-US",
+              "server_vad": true,
+              "dump": true,
+              "max_history": 10
+            }
+          },
+          {
+            "type": "extension",
+            "name": "message_collector",
+            "addon": "message_collector",
+            "extension_group": "transcriber",
+            "property": {}
+          },
+          {
+            "type": "extension",
+            "name": "bingsearch_tool_python",
+            "addon": "bingsearch_tool_python",
+            "extension_group": "default",
+            "property": {
+              "api_key": "${env:BING_API_KEY|}"
+            }
+          },
+          {
+            "type": "extension",
+            "name": "weatherapi_tool_python",
+            "addon": "weatherapi_tool_python",
+            "extension_group": "default",
+            "property": {
+              "api_key": "${env:WEATHERAPI_API_KEY|}"
+            }
+          }
+        ],
+        "connections": [
+          {
+            "extension": "agora_rtc",
+            "cmd": [
+              {
+                "name": "on_user_joined",
+                "dest": [
+                  {
+                    "extension": "v2v"
+                  }
+                ]
+              },
+              {
+                "name": "on_user_left",
+                "dest": [
+                  {
+                    "extension": "v2v"
+                  }
+                ]
+              },
+              {
+                "name": "on_connection_failure",
+                "dest": [
+                  {
+                    "extension": "v2v"
+                  }
+                ]
+              }
+            ],
+            "audio_frame": [
+              {
+                "name": "pcm_frame",
+                "dest": [
+                  {
+                    "extension": "v2v"
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "extension": "v2v",
+            "cmd": [
+              {
+                "name": "flush",
+                "dest": [
+                  {
+                    "extension": "agora_rtc"
+                  }
+                ]
+              },
+              {
+                "name": "tool_call",
+                "dest": [
+                  {
+                    "extension": "bingsearch_tool_python"
+                  },
+                  {
+                    "extension": "weatherapi_tool_python"
+                  }
+                ]
+              }
+            ],
+            "data": [
+              {
+                "name": "text_data",
+                "dest": [
+                  {
+                    "extension": "message_collector"
+                  }
+                ]
+              }
+            ],
+            "audio_frame": [
+              {
+                "name": "pcm_frame",
+                "dest": [
+                  {
+                    "extension": "agora_rtc"
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "extension": "message_collector",
+            "data": [
+              {
+                "name": "data",
+                "dest": [
+                  {
+                    "extension": "agora_rtc"
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "extension": "bingsearch_tool_python",
+            "cmd": [
+              {
+                "name": "tool_register",
+                "dest": [
+                  {
+                    "extension": "v2v"
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "extension": "weatherapi_tool_python",
+            "cmd": [
+              {
+                "name": "tool_register",
+                "dest": [
+                  {
+                    "extension": "v2v"
+                  }
+                ]
+              }
+            ]
+          }
+        ]
+      },
+      {
+        "name": "va_openai_v2v_fish",
+        "auto_start": true,
+        "nodes": [
+          {
+            "type": "extension",
+            "name": "agora_rtc",
+            "addon": "agora_rtc",
+            "extension_group": "rtc",
+            "property": {
+              "app_id": "${env:AGORA_APP_ID}",
+              "token": "",
+              "channel": "ten_agent_test",
+              "stream_id": 1234,
+              "remote_stream_id": 123,
+              "subscribe_audio": true,
+              "publish_audio": true,
+              "publish_data": true,
+              "subscribe_audio_sample_rate": 24000,
+              "enable_agora_asr": false,
+              "agora_asr_vendor_name": "microsoft",
+              "agora_asr_language": "en-US",
+              "agora_asr_vendor_key": "${env:AZURE_STT_KEY}",
+              "agora_asr_vendor_region": "${env:AZURE_STT_REGION}",
+              "agora_asr_session_control_file_path": "session_control.conf"
+            }
+          },
+          {
+            "type": "extension",
+            "name": "v2v",
+            "addon": "openai_v2v_python",
+            "extension_group": "llm",
+            "property": {
+              "api_key": "${env:OPENAI_REALTIME_API_KEY}",
+              "temperature": 0.9,
+              "model": "gpt-4o-realtime-preview-2024-12-17",
+              "max_tokens": 2048,
+              "audio_out": false,
+              "input_transcript": false,
+              "language": "en-US",
+              "server_vad": true,
+              "dump": true,
+              "max_history": 10
+            }
+          },
+          {
+            "type": "extension",
+            "name": "tts",
+            "addon": "fish_audio_tts",
+            "extension_group": "tts",
+            "property": {
+              "api_key": "${env:FISH_AUDIO_TTS_KEY}",
+              "base_url": "https://api.fish.audio",
+              "model_id": "d8639b5cc95548f5afbcfe22d3ba5ce5",
+              "optimize_streaming_latency": true,
+              "request_timeout_seconds": 30
+            }
+          },
+          {
+            "type": "extension",
+            "name": "message_collector",
+            "addon": "message_collector",
+            "extension_group": "transcriber",
+            "property": {}
+          },
+          {
+            "type": "extension",
+            "name": "weatherapi_tool_python",
+            "addon": "weatherapi_tool_python",
+            "extension_group": "tools",
+            "property": {
+              "api_key": "${env:WEATHERAPI_API_KEY}"
+            }
+          },
+          {
+            "type": "extension",
+            "name": "bingsearch_tool_python",
+            "addon": "bingsearch_tool_python",
+            "extension_group": "tools",
+            "property": {
+              "api_key": "${env:BING_API_KEY}"
+            }
+          }
+        ],
+        "connections": [
+          {
+            "extension": "agora_rtc",
+            "data": [
+              {
+                "name": "text_data",
+                "dest": [
+                  {
+                    "extension": "message_collector"
+                  }
+                ]
+              }
+            ],
+            "audio_frame": [
+              {
+                "name": "pcm_frame",
+                "dest": [
+                  {
+                    "extension": "v2v"
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "extension": "weatherapi_tool_python",
+            "cmd": [
+              {
+                "name": "tool_register",
+                "dest": [
+                  {
+                    "extension": "v2v"
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "extension": "bingsearch_tool_python",
+            "cmd": [
+              {
+                "name": "tool_register",
+                "dest": [
+                  {
+                    "extension": "v2v"
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "extension": "v2v",
+            "cmd": [
+              {
+                "name": "flush",
+                "dest": [
+                  {
+                    "extension": "tts"
+                  }
+                ]
+              },
+              {
+                "name": "tool_call",
+                "dest": [
+                  {
+                    "extension": "weatherapi_tool_python"
+                  },
+                  {
+                    "extension": "bingsearch_tool_python"
+                  }
+                ]
+              },
+              {
+                "name": "on_user_joined",
+                "dest": [
+                  {
+                    "extension": "v2v"
+                  }
+                ]
+              },
+              {
+                "name": "on_user_left",
+                "dest": [
+                  {
+                    "extension": "v2v"
+                  }
+                ]
+              }
+            ],
+            "data": [
+              {
+                "name": "text_data",
+                "dest": [
+                  {
+                    "extension": "message_collector"
+                  },
+                  {
+                    "extension": "tts"
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "extension": "tts",
+            "cmd": [
+              {
+                "name": "flush",
+                "dest": [
+                  {
+                    "extension": "agora_rtc"
+                  }
+                ]
+              }
+            ],
+            "audio_frame": [
+              {
+                "name": "pcm_frame",
+                "dest": [
+                  {
+                    "extension": "agora_rtc"
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "extension": "message_collector",
+            "data": [
+              {
+                "name": "data",
+                "dest": [
+                  {
+                    "extension": "agora_rtc"
+                  }
+                ]
+              }
+            ]
+          }
+        ]
+      },
+      {
+        "name": "va_coze_azure",
+        "auto_start": false,
+        "nodes": [
+          {
+            "type": "extension",
+            "name": "agora_rtc",
+            "addon": "agora_rtc",
+            "extension_group": "default",
+            "property": {
+              "app_id": "${env:AGORA_APP_ID}",
+              "token": "<agora_token>",
+              "channel": "ten_agent_test",
+              "stream_id": 1234,
+              "remote_stream_id": 123,
+              "subscribe_audio": true,
+              "publish_audio": true,
+              "publish_data": true,
+              "enable_agora_asr": true,
+              "agora_asr_vendor_name": "microsoft",
+              "agora_asr_language": "en-US",
+              "agora_asr_vendor_key": "${env:AZURE_STT_KEY}",
+              "agora_asr_vendor_region": "${env:AZURE_STT_REGION}",
+              "agora_asr_session_control_file_path": "session_control.conf"
+            }
+          },
+          {
+            "type": "extension",
+            "name": "interrupt_detector",
+            "addon": "interrupt_detector_python",
+            "extension_group": "default"
+          },
+          {
+            "type": "extension",
+            "name": "coze_python_async",
+            "addon": "coze_python_async",
+            "extension_group": "glue",
+            "property": {
+              "token": "<coze_token>",
+              "bot_id": "<coze_bot_id>",
+              "base_url": "https://api.coze.cn",
+              "prompt": "",
+              "greeting": "TEN Agent connected. How can I help you today?"
+            }
+          },
+          {
+            "type": "extension",
+            "name": "tts",
+            "addon": "azure_tts",
+            "extension_group": "tts",
+            "property": {
+              "azure_subscription_key": "${env:AZURE_TTS_KEY}",
+              "azure_subscription_region": "${env:AZURE_TTS_REGION}",
+              "azure_synthesis_voice_name": "en-US-AndrewMultilingualNeural"
+            }
+          },
+          {
+            "type": "extension",
+            "name": "message_collector",
+            "addon": "message_collector",
+            "extension_group": "transcriber"
+          }
+        ],
+        "connections": [
+          {
+            "extension": "agora_rtc",
+            "cmd": [
+              {
+                "name": "on_user_joined",
+                "dest": [
+                  {
+                    "extension": "coze_python_async"
+                  }
+                ]
+              },
+              {
+                "name": "on_user_left",
+                "dest": [
+                  {
+                    "extension": "coze_python_async"
+                  }
+                ]
+              }
+            ],
+            "data": [
+              {
+                "name": "text_data",
+                "dest": [
+                  {
+                    "extension": "interrupt_detector"
+                  },
+                  {
+                    "extension": "coze_python_async"
+                  },
+                  {
+                    "extension": "message_collector"
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "extension": "coze_python_async",
+            "cmd": [
+              {
+                "name": "flush",
+                "dest": [
+                  {
+                    "extension": "tts"
+                  }
+                ]
+              }
+            ],
+            "data": [
+              {
+                "name": "text_data",
+                "dest": [
+                  {
+                    "extension": "tts"
+                  },
+                  {
+                    "extension": "message_collector"
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "extension": "tts",
+            "cmd": [
+              {
+                "name": "flush",
+                "dest": [
+                  {
+                    "extension": "agora_rtc"
+                  }
+                ]
+              }
+            ],
+            "audio_frame": [
+              {
+                "name": "pcm_frame",
+                "dest": [
+                  {
+                    "extension": "agora_rtc"
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "extension": "message_collector",
+            "data": [
+              {
+                "name": "data",
+                "dest": [
+                  {
+                    "extension": "agora_rtc"
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "extension": "interrupt_detector",
+            "cmd": [
+              {
+                "name": "flush",
+                "dest": [
+                  {
+                    "extension": "coze_python_async"
+                  }
+                ]
+              }
+            ]
+          }
+        ]
+      },
+      {
+        "name": "va_gemini_v2v",
+        "auto_start": true,
+        "nodes": [
+          {
+            "type": "extension",
+            "name": "agora_rtc",
+            "addon": "agora_rtc",
+            "extension_group": "rtc",
+            "property": {
+              "app_id": "${env:AGORA_APP_ID}",
+              "token": "",
+              "channel": "ten_agent_test",
+              "stream_id": 1234,
+              "remote_stream_id": 123,
+              "subscribe_audio": true,
+              "publish_audio": true,
+              "publish_data": true,
+              "subscribe_audio_sample_rate": 24000,
+              "subscribe_video_pix_fmt": 4,
+              "subscribe_video": true
+            }
+          },
+          {
+            "type": "extension",
+            "name": "v2v",
+            "addon": "gemini_v2v_python",
+            "extension_group": "llm",
+            "property": {
+              "api_key": "${env:GEMINI_API_KEY}",
+              "api_version": "v1alpha",
+              "base_uri": "generativelanguage.googleapis.com",
+              "dump": true,
+              "language": "en-US",
+              "max_tokens": 2048,
+              "model": "gemini-2.0-flash-exp",
+              "server_vad": true,
+              "temperature": 0.9,
+              "voice": "Puck"
+            }
+          },
+          {
+            "type": "extension",
+            "name": "message_collector",
+            "addon": "message_collector",
+            "extension_group": "transcriber",
+            "property": {}
+          },
+          {
+            "type": "extension",
+            "name": "weatherapi_tool_python",
+            "addon": "weatherapi_tool_python",
+            "extension_group": "default",
+            "property": {
+              "api_key": "${env:WEATHERAPI_API_KEY|}"
+            }
+          }
+        ],
+        "connections": [
+          {
+            "extension": "agora_rtc",
+            "cmd": [
+              {
+                "name": "on_user_joined",
+                "dest": [
+                  {
+                    "extension": "v2v"
+                  }
+                ]
+              },
+              {
+                "name": "on_user_left",
+                "dest": [
+                  {
+                    "extension": "v2v"
+                  }
+                ]
+              },
+              {
+                "name": "on_connection_failure",
+                "dest": [
+                  {
+                    "extension": "v2v"
+                  }
+                ]
+              }
+            ],
+            "audio_frame": [
+              {
+                "name": "pcm_frame",
+                "dest": [
+                  {
+                    "extension": "v2v"
+                  }
+                ]
+              }
+            ],
+            "video_frame": [
+              {
+                "name": "video_frame",
+                "dest": [
+                  {
+                    "extension": "v2v"
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "extension": "v2v",
+            "cmd": [
+              {
+                "name": "flush",
+                "dest": [
+                  {
+                    "extension": "agora_rtc"
+                  }
+                ]
+              },
+              {
+                "name": "tool_call",
+                "dest": [
+                  {
+                    "extension": "weatherapi_tool_python"
+                  }
+                ]
+              }
+            ],
+            "data": [
+              {
+                "name": "text_data",
+                "dest": [
+                  {
+                    "extension": "message_collector"
+                  }
+                ]
+              }
+            ],
+            "audio_frame": [
+              {
+                "name": "pcm_frame",
+                "dest": [
+                  {
+                    "extension": "agora_rtc"
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "extension": "message_collector",
+            "data": [
+              {
+                "name": "data",
+                "dest": [
+                  {
+                    "extension": "agora_rtc"
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "extension": "weatherapi_tool_python",
+            "cmd": [
+              {
+                "name": "tool_register",
+                "dest": [
+                  {
+                    "extension": "v2v"
+                  }
+                ]
+              }
+            ]
+          }
+        ]
+      },
+      {
+        "name": "va_dify_azure",
+        "auto_start": true,
+        "nodes": [
+          {
+            "type": "extension",
+            "name": "agora_rtc",
+            "addon": "agora_rtc",
+            "extension_group": "default",
+            "property": {
+              "app_id": "${env:AGORA_APP_ID}",
+              "token": "<agora_token>",
+              "channel": "ten_agent_test",
+              "stream_id": 1234,
+              "remote_stream_id": 123,
+              "subscribe_audio": true,
+              "publish_audio": true,
+              "publish_data": true,
+              "enable_agora_asr": true,
+              "agora_asr_vendor_name": "microsoft",
+              "agora_asr_language": "en-US",
+              "agora_asr_vendor_key": "${env:AZURE_STT_KEY|}",
+              "agora_asr_vendor_region": "${env:AZURE_STT_REGION|}",
+              "agora_asr_session_control_file_path": "session_control.conf"
+            }
+          },
+          {
+            "type": "extension",
+            "name": "llm",
+            "addon": "dify_python",
+            "extension_group": "chatgpt",
+            "property": {
+              "api_key": "${env:DIFY_API_KEY}",
+              "base_url": "https://api.dify.ai/v1",
+              "greeting": "TEN Agent connected with Dify. How can I help you today?",
+              "user_id": "User"
+            }
+          },
+          {
+            "type": "extension",
+            "name": "tts",
+            "addon": "azure_tts",
+            "extension_group": "tts",
+            "property": {
+              "azure_subscription_key": "${env:AZURE_TTS_KEY}",
+              "azure_subscription_region": "${env:AZURE_TTS_REGION}",
+              "azure_synthesis_voice_name": "en-US-AndrewMultilingualNeural"
+            }
+          },
+          {
+            "type": "extension",
+            "name": "interrupt_detector",
+            "addon": "interrupt_detector_python",
+            "extension_group": "default",
+            "property": {}
+          },
+          {
+            "type": "extension",
+            "name": "message_collector",
+            "addon": "message_collector",
+            "extension_group": "transcriber",
+            "property": {}
+          }
+        ],
+        "connections": [
+          {
+            "extension": "agora_rtc",
+            "cmd": [
+              {
+                "name": "on_user_joined",
+                "dest": [
+                  {
+                    "extension": "llm"
+                  }
+                ]
+              },
+              {
+                "name": "on_user_left",
+                "dest": [
+                  {
+                    "extension": "llm"
+                  }
+                ]
+              },
+              {
+                "name": "on_connection_failure",
+                "dest": [
+                  {
+                    "extension": "llm"
+                  }
+                ]
+              }
+            ],
+            "data": [
+              {
+                "name": "text_data",
+                "dest": [
+                  {
+                    "extension": "interrupt_detector"
+                  },
+                  {
+                    "extension": "message_collector"
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "extension": "llm",
+            "cmd": [
+              {
+                "name": "flush",
+                "dest": [
+                  {
+                    "extension": "tts"
+                  }
+                ]
+              }
+            ],
+            "data": [
+              {
+                "name": "text_data",
+                "dest": [
+                  {
+                    "extension": "tts"
+                  },
+                  {
+                    "extension": "message_collector"
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "extension": "message_collector",
+            "data": [
+              {
+                "name": "data",
+                "dest": [
+                  {
+                    "extension": "agora_rtc"
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "extension": "tts",
+            "cmd": [
+              {
+                "name": "flush",
+                "dest": [
+                  {
+                    "extension": "agora_rtc"
+                  }
+                ]
+              }
+            ],
+            "audio_frame": [
+              {
+                "name": "pcm_frame",
+                "dest": [
+                  {
+                    "extension": "agora_rtc"
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "extension": "interrupt_detector",
+            "cmd": [
+              {
+                "name": "flush",
+                "dest": [
+                  {
+                    "extension": "llm"
+                  }
+                ]
+              }
+            ],
+            "data": [
+              {
+                "name": "text_data",
+                "dest": [
+                  {
+                    "extension": "llm"
+                  }
+                ]
+              }
+            ]
+          }
+        ]
+      },
+      {
+        "name": "story_teller_stt_integrated",
+        "auto_start": true,
+        "nodes": [
+          {
+            "type": "extension",
+            "name": "agora_rtc",
+            "addon": "agora_rtc",
+            "extension_group": "default",
+            "property": {
+              "app_id": "${env:AGORA_APP_ID}",
+              "token": "<agora_token>",
+              "channel": "ten_agent_test",
+              "stream_id": 1234,
+              "remote_stream_id": 123,
+              "subscribe_audio": true,
+              "publish_audio": true,
+              "publish_data": true,
+              "enable_agora_asr": true,
+              "agora_asr_vendor_name": "microsoft",
+              "agora_asr_language": "en-US",
+              "agora_asr_vendor_key": "${env:AZURE_STT_KEY|}",
+              "agora_asr_vendor_region": "${env:AZURE_STT_REGION|}",
+              "agora_asr_session_control_file_path": "session_control.conf"
+            }
+          },
+          {
+            "type": "extension",
+            "name": "llm",
+            "addon": "openai_chatgpt_python",
+            "extension_group": "chatgpt",
+            "property": {
+              "api_key": "${env:OPENAI_API_KEY}",
+              "base_url": "",
+              "frequency_penalty": 0.9,
+              "greeting": "TEN Agent connected. How can I help you today?",
+              "max_memory_length": 10,
+              "max_tokens": 512,
+              "model": "${env:OPENAI_MODEL}",
+              "prompt": "You are an ai agent bot producing child picture books. Each response should be short and no more than 50 words as it's for child. \nFor every response relevant to the story-telling, you will use the 'image_generate' tool to create an image based on the description or key moment in that part of the story. \n The story should be set in a fantasy world. Try asking questions relevant to the story to decide how the story should proceed. Every response should include rich, vivid descriptions that will guide the 'image_generate' tool to produce an image that aligns with the scene or mood.\n Whether it’s the setting, a character’s expression, or a dramatic moment, the paragraph should give enough detail for a meaningful visual representation.",
+              "proxy_url": "${env:OPENAI_PROXY_URL}"
+            }
+          },
+          {
+            "type": "extension",
+            "name": "tts",
+            "addon": "azure_tts",
+            "extension_group": "tts",
+            "property": {
+              "azure_subscription_key": "${env:AZURE_TTS_KEY}",
+              "azure_subscription_region": "${env:AZURE_TTS_REGION}",
+              "azure_synthesis_voice_name": "en-US-AndrewMultilingualNeural"
+            }
+          },
+          {
+            "type": "extension",
+            "name": "interrupt_detector",
+            "addon": "interrupt_detector_python",
+            "extension_group": "default",
+            "property": {}
+          },
+          {
+            "type": "extension",
+            "name": "message_collector",
+            "addon": "message_collector",
+            "extension_group": "transcriber",
+            "property": {}
+          },
+          {
+            "type": "extension",
+            "name": "openai_image_generate_tool",
+            "addon": "openai_image_generate_tool",
+            "extension_group": "default",
+            "property": {
+              "api_key": "${env:OPENAI_API_KEY}"
+            }
+          }
+        ],
+        "connections": [
+          {
+            "extension": "agora_rtc",
+            "cmd": [
+              {
+                "name": "on_user_joined",
+                "dest": [
+                  {
+                    "extension": "llm"
+                  }
+                ]
+              },
+              {
+                "name": "on_user_left",
+                "dest": [
+                  {
+                    "extension": "llm"
+                  }
+                ]
+              },
+              {
+                "name": "on_connection_failure",
+                "dest": [
+                  {
+                    "extension": "llm"
+                  }
+                ]
+              }
+            ],
+            "data": [
+              {
+                "name": "text_data",
+                "dest": [
+                  {
+                    "extension": "interrupt_detector"
+                  },
+                  {
+                    "extension": "message_collector"
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "extension": "llm",
+            "cmd": [
+              {
+                "name": "flush",
+                "dest": [
+                  {
+                    "extension": "tts"
+                  }
+                ]
+              },
+              {
+                "name": "tool_call",
+                "dest": [
+                  {
+                    "extension": "openai_image_generate_tool"
+                  }
+                ]
+              }
+            ],
+            "data": [
+              {
+                "name": "text_data",
+                "dest": [
+                  {
+                    "extension": "tts"
+                  },
+                  {
+                    "extension": "message_collector"
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "extension": "message_collector",
+            "data": [
+              {
+                "name": "data",
+                "dest": [
+                  {
+                    "extension": "agora_rtc"
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "extension": "tts",
+            "cmd": [
+              {
+                "name": "flush",
+                "dest": [
+                  {
+                    "extension": "agora_rtc"
+                  }
+                ]
+              }
+            ],
+            "audio_frame": [
+              {
+                "name": "pcm_frame",
+                "dest": [
+                  {
+                    "extension": "agora_rtc"
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "extension": "interrupt_detector",
+            "cmd": [
+              {
+                "name": "flush",
+                "dest": [
+                  {
+                    "extension": "llm"
+                  }
+                ]
+              }
+            ],
+            "data": [
+              {
+                "name": "text_data",
+                "dest": [
+                  {
+                    "extension": "llm"
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "extension": "openai_image_generate_tool",
+            "cmd": [
+              {
+                "name": "tool_register",
+                "dest": [
+                  {
+                    "extension": "llm"
+                  }
+                ]
+              }
+            ],
+            "data": [
+              {
+                "name": "content_data",
+                "dest": [
+                  {
+                    "extension": "message_collector"
+                  }
+                ]
+              }
+            ]
+          }
+        ]
+      },
+      {
+        "name": "va_nova_multimodal_aws",
+        "auto_start": true,
+        "nodes": [
+          {
+            "type": "extension",
+            "name": "agora_rtc",
+            "addon": "agora_rtc",
+            "extension_group": "default",
+            "property": {
+              "app_id": "${env:AGORA_APP_ID}",
+              "token": "<agora_token>",
+              "channel": "ten_agent_test",
+              "stream_id": 1234,
+              "remote_stream_id": 123,
+              "subscribe_audio": true,
+              "publish_audio": true,
+              "publish_data": true,
+              "enable_agora_asr": false,
+              "agora_asr_vendor_name": "microsoft",
+              "agora_asr_language": "en-US",
+              "agora_asr_vendor_key": "${env:AZURE_STT_KEY|}",
+              "agora_asr_vendor_region": "${env:AZURE_STT_REGION|}",
+              "agora_asr_session_control_file_path": "session_control.conf",
+              "subscribe_video_pix_fmt": 4,
+              "subscribe_video": true,
+              "max_memory_length": 10
+            }
+          },
+          {
+            "type": "extension",
+            "name": "stt",
+            "addon": "transcribe_asr_python",
+            "extension_group": "stt",
+            "property": {
+              "access_key": "${env:AWS_ACCESS_KEY_ID}",
+              "lang_code": "en-US",
+              "region": "us-east-1",
+              "sample_rate": "16000",
+              "secret_key": "${env:AWS_SECRET_ACCESS_KEY}"
+            }
+          },
+          {
+            "type": "extension",
+            "name": "llm",
+            "addon": "bedrock_llm_python",
+            "extension_group": "chatgpt",
+            "property": {
+              "access_key_id": "${env:AWS_ACCESS_KEY_ID}",
+              "greeting": "TEN Agent connected. I am nova, How can I help you today?",
+              "max_memory_length": 10,
+              "max_tokens": 256,
+              "model": "us.amazon.nova-lite-v1:0",
+              "prompt": "Now you are an intelligent assistant with real-time interaction capabilities. I will provide you with a series of real-time video image information. Please understand these images as video frames. Based on the images and the user's input, engage in a conversation with the user, remembering the dialogue content in a concise and clear manner.",
+              "region": "us-east-1",
+              "secret_access_key": "${env:AWS_SECRET_ACCESS_KEY}",
+              "temperature": 0.7,
+              "topK": 10,
+              "topP": 0.5,
+              "is_memory_enabled": false,
+              "is_enable_video": true
+            }
+          },
+          {
+            "type": "extension",
+            "name": "tts",
+            "addon": "polly_tts",
+            "extension_group": "tts",
+            "property": {
+              "region": "us-east-1",
+              "access_key": "${env:AWS_ACCESS_KEY_ID}",
+              "secret_key": "${env:AWS_SECRET_ACCESS_KEY}",
+              "engine": "generative",
+              "voice": "Ruth",
+              "sample_rate": 16000,
+              "lang_code": "en-US"
+            }
+          },
+          {
+            "type": "extension",
+            "name": "interrupt_detector",
+            "addon": "interrupt_detector_python",
+            "extension_group": "default",
+            "property": {}
+          },
+          {
+            "type": "extension",
+            "name": "message_collector",
+            "addon": "message_collector",
+            "extension_group": "transcriber",
+            "property": {}
+          }
+        ],
+        "connections": [
+          {
+            "extension": "agora_rtc",
+            "cmd": [
+              {
+                "name": "on_user_joined",
+                "dest": [
+                  {
+                    "extension": "llm"
+                  }
+                ]
+              },
+              {
+                "name": "on_user_left",
+                "dest": [
+                  {
+                    "extension": "llm"
+                  }
+                ]
+              },
+              {
+                "name": "on_connection_failure",
+                "dest": [
+                  {
+                    "extension": "llm"
+                  }
+                ]
+              }
+            ],
+            "audio_frame": [
+              {
+                "name": "pcm_frame",
+                "dest": [
+                  {
+                    "extension": "stt"
+                  }
+                ]
+              }
+            ],
+            "video_frame": [
+              {
+                "name": "video_frame",
+                "dest": [
+                  {
+                    "extension": "llm"
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "extension": "stt",
+            "data": [
+              {
+                "name": "text_data",
+                "dest": [
+                  {
+                    "extension": "interrupt_detector"
+                  },
+                  {
+                    "extension": "message_collector"
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "extension": "llm",
+            "cmd": [
+              {
+                "name": "flush",
+                "dest": [
+                  {
+                    "extension": "tts"
+                  }
+                ]
+              }
+            ],
+            "data": [
+              {
+                "name": "text_data",
+                "dest": [
+                  {
+                    "extension": "tts"
+                  },
+                  {
+                    "extension": "message_collector"
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "extension": "message_collector",
+            "data": [
+              {
+                "name": "data",
+                "dest": [
+                  {
+                    "extension": "agora_rtc"
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "extension": "tts",
+            "cmd": [
+              {
+                "name": "flush",
+                "dest": [
+                  {
+                    "extension": "agora_rtc"
+                  }
+                ]
+              }
+            ],
+            "audio_frame": [
+              {
+                "name": "pcm_frame",
+                "dest": [
+                  {
+                    "extension": "agora_rtc"
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "extension": "interrupt_detector",
+            "cmd": [
+              {
+                "name": "flush",
+                "dest": [
+                  {
+                    "extension": "llm"
+                  }
+                ]
+              }
+            ],
+            "data": [
+              {
+                "name": "text_data",
+                "dest": [
+                  {
+                    "extension": "llm"
+                  }
+                ]
+              }
+            ]
+          }
+        ]
+      }
+    ],
+    "log_level": 3
+  }
+}

agents/examples/experimental/manifest.json

@@ -0,0 +1,122 @@
+{
+  "type": "app",
+  "name": "agent_experimental",
+  "version": "0.8.0",
+  "dependencies": [
+    {
+      "type": "system",
+      "name": "ten_runtime_go",
+      "version": "0.8"
+    },
+    {
+      "type": "extension",
+      "name": "agora_rtc",
+      "version": "=0.12.0"
+    },
+    {
+      "type": "extension",
+      "name": "agora_sess_ctrl",
+      "version": "=0.4.4"
+    },
+    {
+      "type": "system",
+      "name": "azure_speech_sdk",
+      "version": "1.38.0"
+    },
+    {
+      "type": "system",
+      "name": "ten_ai_base",
+      "version": "0.4.1"
+    },
+    {
+      "type": "extension",
+      "name": "azure_tts",
+      "version": "=0.8.1"
+    },
+    {
+      "type": "extension",
+      "name": "agora_rtm",
+      "version": "=0.8.1"
+    },
+    {
+      "type": "extension",
+      "name": "interrupt_detector_python",
+      "version": "=0.1.0"
+    },
+    {
+      "type": "extension",
+      "name": "openai_chatgpt_python",
+      "version": "=0.1.0"
+    },
+    {
+      "type": "extension",
+      "name": "message_collector",
+      "version": "=0.1.0"
+    },
+    {
+      "type": "extension",
+      "name": "fashionai",
+      "version": "=0.1.0"
+    },
+    {
+      "type": "extension",
+      "name": "qwen_llm_python",
+      "version": "=0.1.0"
+    },
+    {
+      "type": "extension",
+      "name": "cosy_tts_python",
+      "version": "=0.1.0"
+    },
+    {
+      "type": "extension",
+      "name": "http_server_python",
+      "version": "=0.10.1"
+    },
+    {
+      "type": "extension",
+      "name": "aliyun_text_embedding",
+      "version": "=0.1.0"
+    },
+    {
+      "type": "extension",
+      "name": "aliyun_analyticdb_vector_storage",
+      "version": "=0.1.0"
+    },
+    {
+      "type": "extension",
+      "name": "file_chunker",
+      "version": "=0.1.0"
+    },
+    {
+      "type": "extension",
+      "name": "llama_index_chat_engine",
+      "version": "=0.1.0"
+    },
+    {
+      "type": "extension",
+      "name": "openai_v2v_python",
+      "version": "=0.1.0"
+    },
+    {
+      "type": "extension",
+      "name": "weatherapi_tool_python",
+      "version": "=0.1.0"
+    },
+    {
+      "type": "extension",
+      "name": "bingsearch_tool_python",
+      "version": "=0.1.0"
+    },
+    {
+      "type": "extension",
+      "name": "tsdb_firestore",
+      "version": "=0.1.0"
+    },
+    {
+      "type": "extension",
+      "name": "minimax_v2v_python",
+      "version": "=0.1.0"
+    }
+  ]
+}

agents/examples/experimental/property.json

@@ -0,0 +1,862 @@
+{
+  "_ten": {
+    "log_level": 3,
+    "predefined_graphs": [
+      {
+        "name": "va_openai_azure_fashionai",
+        "auto_start": false,
+        "connections": [
+          {
+            "data": [
+              {
+                "dest": [
+                  {
+                    "extension": "interrupt_detector"
+                  },
+                  {
+                    "extension": "openai_chatgpt"
+                  },
+                  {
+                    "extension": "message_collector"
+                  }
+                ],
+                "name": "text_data"
+              }
+            ],
+            "cmd": [
+              {
+                "name": "on_user_joined",
+                "dest": [
+                  {
+                    "extension": "openai_chatgpt"
+                  }
+                ]
+              },
+              {
+                "name": "on_user_left",
+                "dest": [
+                  {
+                    "extension": "openai_chatgpt"
+                  }
+                ]
+              }
+            ],
+            "extension": "agora_rtc"
+          },
+          {
+            "cmd": [
+              {
+                "dest": [
+                  {
+                    "extension": "fashionai"
+                  }
+                ],
+                "name": "flush"
+              }
+            ],
+            "data": [
+              {
+                "dest": [
+                  {
+                    "extension": "message_collector"
+                  },
+                  {
+                    "extension": "fashionai"
+                  }
+                ],
+                "name": "text_data"
+              }
+            ],
+            "extension": "openai_chatgpt"
+          },
+          {
+            "data": [
+              {
+                "dest": [
+                  {
+                    "extension": "agora_rtc"
+                  }
+                ],
+                "name": "data"
+              }
+            ],
+            "extension": "message_collector"
+          },
+          {
+            "cmd": [
+              {
+                "dest": [
+                  {
+                    "extension": "openai_chatgpt"
+                  }
+                ],
+                "name": "flush"
+              }
+            ],
+            "extension": "interrupt_detector"
+          }
+        ],
+        "nodes": [
+          {
+            "addon": "agora_rtc",
+            "extension_group": "default",
+            "name": "agora_rtc",
+            "property": {
+              "agora_asr_language": "en-US",
+              "agora_asr_session_control_file_path": "session_control.conf",
+              "agora_asr_vendor_key": "${env:AZURE_STT_KEY}",
+              "agora_asr_vendor_name": "microsoft",
+              "agora_asr_vendor_region": "${env:AZURE_STT_REGION}",
+              "app_id": "${env:AGORA_APP_ID}",
+              "channel": "ten_agent_test",
+              "enable_agora_asr": true,
+              "publish_audio": true,
+              "publish_data": true,
+              "remote_stream_id": 123,
+              "stream_id": 1234,
+              "subscribe_audio": true,
+              "token": "<agora_token>"
+            },
+            "type": "extension"
+          },
+          {
+            "addon": "interrupt_detector",
+            "extension_group": "default",
+            "name": "interrupt_detector",
+            "type": "extension"
+          },
+          {
+            "addon": "openai_chatgpt_python",
+            "extension_group": "chatgpt",
+            "name": "openai_chatgpt",
+            "property": {
+              "api_key": "${env:OPENAI_API_KEY}",
+              "base_url": "${env:OPENAI_API_BASE}",
+              "frequency_penalty": 0.9,
+              "greeting": "TEN Agent connected. How can I help you today?",
+              "max_memory_length": 10,
+              "max_tokens": 512,
+              "model": "${env:OPENAI_MODEL}",
+              "prompt": "",
+              "proxy_url": "${env:OPENAI_PROXY_URL}"
+            },
+            "type": "extension"
+          },
+          {
+            "addon": "message_collector",
+            "extension_group": "transcriber",
+            "name": "message_collector",
+            "type": "extension"
+          },
+          {
+            "addon": "fashionai",
+            "extension_group": "default",
+            "name": "fashionai",
+            "property": {
+              "app_id": "${env:AGORA_APP_ID}",
+              "channel": "ten_agent_test",
+              "stream_id": 12345,
+              "token": "<agora_token>",
+              "service_id": "agoramultimodel"
+            },
+            "type": "extension"
+          }
+        ]
+      },
+      {
+        "name": "va_qwen_rag",
+        "auto_start": false,
+        "nodes": [
+          {
+            "type": "extension",
+            "extension_group": "rtc",
+            "addon": "agora_rtc",
+            "name": "agora_rtc",
+            "property": {
+              "app_id": "${env:AGORA_APP_ID}",
+              "token": "<agora_token>",
+              "channel": "ten_agent_test",
+              "stream_id": 1234,
+              "remote_stream_id": 123,
+              "subscribe_audio": true,
+              "publish_audio": true,
+              "publish_data": true,
+              "enable_agora_asr": true,
+              "agora_asr_vendor_name": "microsoft",
+              "agora_asr_language": "en-US",
+              "agora_asr_vendor_key": "${env:AZURE_STT_KEY}",
+              "agora_asr_vendor_region": "${env:AZURE_STT_REGION}",
+              "agora_asr_session_control_file_path": "session_control.conf"
+            }
+          },
+          {
+            "type": "extension",
+            "extension_group": "llm",
+            "addon": "qwen_llm_python",
+            "name": "qwen_llm",
+            "property": {
+              "api_key": "${env:QWEN_API_KEY}",
+              "model": "qwen-max",
+              "max_tokens": 512,
+              "prompt": "",
+              "max_memory_length": 10,
+              "greeting": "TEN Agent connected. How can I help you today?"
+            }
+          },
+          {
+            "type": "extension",
+            "extension_group": "tts",
+            "addon": "cosy_tts_python",
+            "name": "cosy_tts",
+            "property": {
+              "api_key": "${env:QWEN_API_KEY}",
+              "model": "cosyvoice-v1",
+              "voice": "longxiaochun",
+              "sample_rate": 16000
+            }
+          },
+          {
+            "type": "extension",
+            "extension_group": "tts",
+            "addon": "azure_tts",
+            "name": "azure_tts",
+            "property": {
+              "azure_subscription_key": "${env:AZURE_TTS_KEY}",
+              "azure_subscription_region": "${env:AZURE_TTS_REGION}",
+              "azure_synthesis_voice_name": "en-US-AndrewMultilingualNeural"
+            }
+          },
+          {
+            "type": "extension",
+            "extension_group": "chat_transcriber",
+            "addon": "message_collector",
+            "name": "message_collector"
+          },
+          {
+            "type": "extension",
+            "extension_group": "interrupt_detector",
+            "addon": "interrupt_detector_python",
+            "name": "interrupt_detector"
+          },
+          {
+            "type": "extension",
+            "extension_group": "http_server",
+            "addon": "http_server_python",
+            "name": "http_server",
+            "property": {
+              "listen_addr": "127.0.0.1",
+              "listen_port": 8080
+            }
+          },
+          {
+            "type": "extension",
+            "extension_group": "embedding",
+            "addon": "aliyun_text_embedding",
+            "name": "aliyun_text_embedding",
+            "property": {
+              "api_key": "${env:ALIYUN_TEXT_EMBEDDING_API_KEY}",
+              "model": "text-embedding-v3"
+            }
+          },
+          {
+            "type": "extension",
+            "extension_group": "vector_storage",
+            "addon": "aliyun_analyticdb_vector_storage",
+            "name": "aliyun_analyticdb_vector_storage",
+            "property": {
+              "alibaba_cloud_access_key_id": "${env:ALIBABA_CLOUD_ACCESS_KEY_ID}",
+              "alibaba_cloud_access_key_secret": "${env:ALIBABA_CLOUD_ACCESS_KEY_SECRET}",
+              "adbpg_instance_id": "${env:ALIYUN_ANALYTICDB_INSTANCE_ID}",
+              "adbpg_instance_region": "${env:ALIYUN_ANALYTICDB_INSTANCE_REGION}",
+              "adbpg_account": "${env:ALIYUN_ANALYTICDB_ACCOUNT}",
+              "adbpg_account_password": "${env:ALIYUN_ANALYTICDB_ACCOUNT_PASSWORD}",
+              "adbpg_namespace": "${env:ALIYUN_ANALYTICDB_NAMESPACE}",
+              "adbpg_namespace_password": "${env:ALIYUN_ANALYTICDB_NAMESPACE_PASSWORD}"
+            }
+          },
+          {
+            "type": "extension",
+            "extension_group": "file_chunker",
+            "addon": "file_chunker",
+            "name": "file_chunker",
+            "property": {}
+          },
+          {
+            "type": "extension",
+            "extension_group": "llama_index",
+            "addon": "llama_index_chat_engine",
+            "name": "llama_index",
+            "property": {
+              "greeting": "TEN Agent connected. How can I help you today?",
+              "chat_memory_token_limit": 3000
+            }
+          }
+        ],
+        "connections": [
+          {
+            "extension": "agora_rtc",
+            "data": [
+              {
+                "name": "text_data",
+                "dest": [
+                  {
+                    "extension": "interrupt_detector"
+                  },
+                  {
+                    "extension": "message_collector"
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "extension": "interrupt_detector",
+            "cmd": [
+              {
+                "name": "flush",
+                "dest": [
+                  {
+                    "extension": "llama_index"
+                  }
+                ]
+              },
+              {
+                "name": "file_chunk",
+                "dest": [
+                  {
+                    "extension": "file_chunker"
+                  },
+                  {
+                    "extension": "llama_index"
+                  }
+                ]
+              },
+              {
+                "name": "file_chunked",
+                "dest": [
+                  {
+                    "extension": "llama_index"
+                  }
+                ]
+              },
+              {
+                "name": "update_querying_collection",
+                "dest": [
+                  {
+                    "extension": "llama_index"
+                  }
+                ]
+              }
+            ],
+            "data": [
+              {
+                "name": "text_data",
+                "dest": [
+                  {
+                    "extension": "llama_index"
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "extension": "llama_index",
+            "data": [
+              {
+                "name": "text_data",
+                "dest": [
+                  {
+                    "extension": "azure_tts"
+                  },
+                  {
+                    "extension": "message_collector"
+                  }
+                ]
+              }
+            ],
+            "cmd": [
+              {
+                "name": "flush",
+                "dest": [
+                  {
+                    "extension": "qwen_llm"
+                  },
+                  {
+                    "extension": "azure_tts"
+                  }
+                ]
+              },
+              {
+                "name": "call_chat",
+                "dest": [
+                  {
+                    "extension": "qwen_llm"
+                  }
+                ]
+              },
+              {
+                "name": "embed",
+                "dest": [
+                  {
+                    "extension": "aliyun_text_embedding"
+                  }
+                ]
+              },
+              {
+                "name": "query_vector",
+                "dest": [
+                  {
+                    "extension": "aliyun_analyticdb_vector_storage"
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "extension": "azure_tts",
+            "audio_frame": [
+              {
+                "name": "pcm_frame",
+                "dest": [
+                  {
+                    "extension": "agora_rtc"
+                  }
+                ]
+              }
+            ],
+            "cmd": [
+              {
+                "name": "flush",
+                "dest": [
+                  {
+                    "extension": "agora_rtc"
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "extension": "message_collector",
+            "data": [
+              {
+                "name": "data",
+                "dest": [
+                  {
+                    "extension": "agora_rtc"
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "extension": "http_server",
+            "cmd": [
+              {
+                "name": "file_chunk",
+                "dest": [
+                  {
+                    "extension": "interrupt_detector"
+                  }
+                ]
+              },
+              {
+                "name": "update_querying_collection",
+                "dest": [
+                  {
+                    "extension": "interrupt_detector"
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "extension": "file_chunker",
+            "cmd": [
+              {
+                "name": "embed_batch",
+                "dest": [
+                  {
+                    "extension": "aliyun_text_embedding"
+                  }
+                ]
+              },
+              {
+                "name": "create_collection",
+                "dest": [
+                  {
+                    "extension": "aliyun_analyticdb_vector_storage"
+                  }
+                ]
+              },
+              {
+                "name": "upsert_vector",
+                "dest": [
+                  {
+                    "extension": "aliyun_analyticdb_vector_storage"
+                  }
+                ]
+              },
+              {
+                "name": "file_chunked",
+                "dest": [
+                  {
+                    "extension": "llama_index"
+                  }
+                ]
+              }
+            ]
+          }
+        ]
+      },
+      {
+        "name": "va_openai_v2v_storage",
+        "auto_start": false,
+        "nodes": [
+          {
+            "type": "extension",
+            "extension_group": "rtc",
+            "addon": "agora_rtc",
+            "name": "agora_rtc",
+            "property": {
+              "app_id": "${env:AGORA_APP_ID}",
+              "token": "",
+              "channel": "ten_agent_test",
+              "stream_id": 1234,
+              "remote_stream_id": 123,
+              "subscribe_audio": true,
+              "publish_audio": true,
+              "publish_data": true,
+              "subscribe_audio_sample_rate": 24000
+            }
+          },
+          {
+            "type": "extension",
+            "extension_group": "llm",
+            "addon": "openai_v2v_python",
+            "name": "openai_v2v_python",
+            "property": {
+              "api_key": "${env:OPENAI_REALTIME_API_KEY}",
+              "temperature": 0.9,
+              "model": "gpt-4o-realtime-preview-2024-12-17",
+              "max_tokens": 2048,
+              "voice": "alloy",
+              "language": "en-US",
+              "server_vad": true,
+              "dump": true,
+              "max_history": 10,
+              "enable_storage": true
+            }
+          },
+          {
+            "type": "extension",
+            "extension_group": "transcriber",
+            "addon": "message_collector",
+            "name": "message_collector"
+          },
+          {
+            "type": "extension",
+            "extension_group": "tools",
+            "addon": "weatherapi_tool_python",
+            "name": "weatherapi_tool_python",
+            "property": {
+              "api_key": "${env:WEATHERAPI_API_KEY}"
+            }
+          },
+          {
+            "type": "extension",
+            "extension_group": "tools",
+            "addon": "bingsearch_tool_python",
+            "name": "bingsearch_tool_python",
+            "property": {
+              "api_key": "${env:BING_API_KEY}"
+            }
+          },
+          {
+            "type": "extension",
+            "extension_group": "context",
+            "addon": "tsdb_firestore",
+            "name": "tsdb_firestore",
+            "property": {
+              "credentials": {
+                "type": "service_account",
+                "project_id": "${env:FIRESTORE_PROJECT_ID}",
+                "private_key_id": "${env:FIRESTORE_PRIVATE_KEY_ID}",
+                "private_key": "${env:FIRESTORE_PRIVATE_KEY}",
+                "client_email": "${env:FIRESTORE_CLIENT_EMAIL}",
+                "client_id": "${env:FIRESTORE_CLIENT_ID}",
+                "auth_uri": "https://accounts.google.com/o/oauth2/auth",
+                "token_uri": "https://oauth2.googleapis.com/token",
+                "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
+                "client_x509_cert_url": "${env:FIRESTORE_CERT_URL}",
+                "universe_domain": "googleapis.com"
+              },
+              "channel_name": "ten_agent_test",
+              "collection_name": "llm_context"
+            }
+          }
+        ],
+        "connections": [
+          {
+            "extension": "agora_rtc",
+            "audio_frame": [
+              {
+                "name": "pcm_frame",
+                "dest": [
+                  {
+                    "extension": "openai_v2v_python"
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "extension": "weatherapi_tool_python",
+            "cmd": [
+              {
+                "name": "tool_register",
+                "dest": [
+                  {
+                    "extension": "openai_v2v_python"
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "extension": "bingsearch_tool_python",
+            "cmd": [
+              {
+                "name": "tool_register",
+                "dest": [
+                  {
+                    "extension": "openai_v2v_python"
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "extension": "openai_v2v_python",
+            "audio_frame": [
+              {
+                "name": "pcm_frame",
+                "dest": [
+                  {
+                    "extension": "agora_rtc"
+                  }
+                ]
+              }
+            ],
+            "data": [
+              {
+                "name": "append",
+                "dest": [
+                  {
+                    "extension": "tsdb_firestore"
+                  }
+                ]
+              },
+              {
+                "name": "text_data",
+                "dest": [
+                  {
+                    "extension": "message_collector"
+                  }
+                ]
+              }
+            ],
+            "cmd": [
+              {
+                "name": "flush",
+                "dest": [
+                  {
+                    "extension": "agora_rtc"
+                  }
+                ]
+              },
+              {
+                "name": "retrieve",
+                "dest": [
+                  {
+                    "extension": "tsdb_firestore"
+                  }
+                ]
+              },
+              {
+                "name": "tool_call",
+                "dest": [
+                  {
+                    "extension": "weatherapi_tool_python"
+                  }
+                ]
+              },
+              {
+                "name": "tool_call",
+                "dest": [
+                  {
+                    "extension": "weatherapi_tool_python"
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "extension": "message_collector",
+            "data": [
+              {
+                "name": "data",
+                "dest": [
+                  {
+                    "extension": "agora_rtc"
+                  }
+                ]
+              }
+            ]
+          }
+        ]
+      },
+      {
+        "name": "va_minimax_v2v",
+        "auto_start": false,
+        "nodes": [
+          {
+            "type": "extension",
+            "extension_group": "rtc",
+            "addon": "agora_rtc",
+            "name": "agora_rtc",
+            "property": {
+              "app_id": "${env:AGORA_APP_ID}",
+              "token": "",
+              "channel": "ten_agent_test",
+              "stream_id": 1234,
+              "remote_stream_id": 123,
+              "subscribe_audio": true,
+              "publish_audio": true,
+              "publish_data": true
+            }
+          },
+          {
+            "type": "extension",
+            "extension_group": "agora_sess_ctrl",
+            "addon": "agora_sess_ctrl",
+            "name": "agora_sess_ctrl",
+            "property": {
+              "wait_for_eos": true
+            }
+          },
+          {
+            "type": "extension",
+            "extension_group": "llm",
+            "addon": "minimax_v2v_python",
+            "name": "minimax_v2v_python",
+            "property": {
+              "in_sample_rate": 16000,
+              "token": "${env:MINIMAX_TOKEN}"
+            }
+          },
+          {
+            "type": "extension",
+            "extension_group": "message_collector",
+            "addon": "message_collector",
+            "name": "message_collector"
+          }
+        ],
+        "connections": [
+          {
+            "extension": "agora_rtc",
+            "audio_frame": [
+              {
+                "name": "pcm_frame",
+                "dest": [
+                  {
+                    "extension": "agora_sess_ctrl"
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "extension": "agora_sess_ctrl",
+            "audio_frame": [
+              {
+                "name": "pcm_frame",
+                "dest": [
+                  {
+                    "extension": "minimax_v2v_python"
+                  }
+                ]
+              }
+            ],
+            "cmd": [
+              {
+                "name": "start_of_sentence",
+                "dest": [
+                  {
+                    "extension": "minimax_v2v_python",
+                    "msg_conversion": {
+                      "type": "per_property",
+                      "keep_original": true,
+                      "rules": [
+                        {
+                          "path": "_ten.name",
+                          "conversion_mode": "fixed_value",
+                          "value": "flush"
+                        }
+                      ]
+                    }
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "extension": "minimax_v2v_python",
+            "data": [
+              {
+                "name": "text_data",
+                "dest": [
+                  {
+                    "extension": "message_collector"
+                  }
+                ]
+              }
+            ],
+            "audio_frame": [
+              {
+                "name": "pcm_frame",
+                "dest": [
+                  {
+                    "extension": "agora_rtc"
+                  }
+                ]
+              }
+            ],
+            "cmd": [
+              {
+                "name": "flush",
+                "dest": [
+                  {
+                    "extension": "agora_rtc"
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "extension": "message_collector",
+            "data": [
+              {
+                "name": "data",
+                "dest": [
+                  {
+                    "extension": "agora_rtc"
+                  }
+                ]
+              }
+            ]
+          }
+        ]
+      }
+    ]
+  }
+}

agents/go.mod

@@ -0,0 +1,7 @@
+module app
+
+go 1.20
+
+replace ten_framework => ./ten_packages/system/ten_runtime_go/interface
+
+require ten_framework v0.0.0-00010101000000-000000000000

agents/main.go

@@ -0,0 +1,71 @@
+/**
+ *
+ * Agora Real Time Engagement
+ * Created by Wei Hu in 2022-10.
+ * Copyright (c) 2024 Agora IO. All rights reserved.
+ *
+ */
+package main
+
+import (
+	"flag"
+	"log"
+	"os"
+
+	"ten_framework/ten"
+)
+
+type appConfig struct {
+	PropertyFilePath string
+}
+
+type defaultApp struct {
+	ten.DefaultApp
+
+	cfg *appConfig
+}
+
+func (p *defaultApp) OnConfigure(
+	tenEnv ten.TenEnv,
+) {
+	// Using the default property.json if not specified.
+	if len(p.cfg.PropertyFilePath) > 0 {
+		if b, err := os.ReadFile(p.cfg.PropertyFilePath); err != nil {
+			log.Fatalf("Failed to read property file %s, err %v\n", p.cfg.PropertyFilePath, err)
+		} else {
+			tenEnv.InitPropertyFromJSONBytes(b)
+		}
+	}
+
+	tenEnv.OnConfigureDone()
+}
+
+func startAppBlocking(cfg *appConfig) {
+	appInstance, err := ten.NewApp(&defaultApp{
+		cfg: cfg,
+	})
+	if err != nil {
+		log.Fatalf("Failed to create the app, %v\n", err)
+	}
+
+	appInstance.Run(true)
+	appInstance.Wait()
+
+	ten.EnsureCleanupWhenProcessExit()
+}
+
+func setDefaultLog() {
+	log.SetFlags(log.LstdFlags | log.Lmicroseconds)
+}
+
+func main() {
+	// Set the default log format globally, users can use `log.Println()` directly.
+	setDefaultLog()
+
+	cfg := &appConfig{}
+
+	flag.StringVar(&cfg.PropertyFilePath, "property", "", "The absolute path of property.json")
+	flag.Parse()
+
+	startAppBlocking(cfg)
+}

agents/scripts/.gitignore

@@ -0,0 +1,4 @@
+graph
+graph.pdf
+graph.png
+

agents/scripts/BUILD.gn

@@ -0,0 +1,18 @@
+#
+#
+# Agora Real Time Engagement
+# Created by Wei Hu in 2022-11.
+# Copyright (c) 2024 Agora IO. All rights reserved.
+#
+#
+import("//build/feature/ten_package.gni")
+
+ten_package("default_app_go") {
+  package_kind = "app"
+  enable_build = false
+
+  resources = [
+    "manifest.json",
+    "property.json",
+  ]
+}

agents/scripts/dot.py

@@ -0,0 +1,117 @@
+# Generate a graph from a JSON file
+# Currently it only generate the first graph but it's easy to make it generate all graphs
+# Author: Seven Du <[email protected]>
+# usage:
+#     pip install graphviz
+#     python dot.py
+
+import json
+import graphviz
+
+COLORS = {
+    "flush": "#999",
+    "cmd": "#0f0",
+    "data": "#00f",
+    "text_data": "#f00",
+    "pcm_frame": "purple",
+}
+
+connection_types = ["data", "cmd", "audio_frame", "video_frame"]
+
+
+def color(port):
+    if port in COLORS:
+        return COLORS[port]
+    return "#000"
+
+
+def find_node(nodes, name):
+    for node in nodes:
+        if node["name"] == name:
+            return node
+    return None
+
+
+def create_graph(json_data):
+    # Initialize a directed graph
+    graph = graphviz.Digraph("G", filename="graph.gv")
+    graph.graph_attr["rankdir"] = "LR"
+    graph.graph_attr["dpi"] = "150"
+    graph.graph_attr["splines"] = "true"
+    graph.attr("node", shape="none")
+
+    # Add nodes to the graph
+    nodes = json_data["_ten"]["predefined_graphs"][0]["nodes"]
+    connections = json_data["_ten"]["predefined_graphs"][0]["connections"]
+    for node in nodes:
+        node["i_ports"] = ["flush"]
+        node["o_ports"] = ["flush"]
+    for node in nodes:
+        if node["type"] != "extension":
+            continue
+        for connection in connections:
+            if connection["extension"] == node["name"]:
+                for connection_type in connection_types:
+                    if connection_type in connection:
+                        data = connection[connection_type]
+                        for item in data:
+                            node["o_ports"].append(item["name"])
+                            for dest in item["dest"]:
+                                dest_node = find_node(nodes, dest["extension"])
+                                if dest_node:
+                                    dest_node["i_ports"].append(item["name"])
+    for node in nodes:
+        if node["type"] != "extension":
+            continue
+        node["i_ports"] = set(node["i_ports"])
+        node["o_ports"] = set(node["o_ports"])
+        print("====iports: ", node["name"], node["i_ports"])
+        print("====oports: ", node["name"], node["o_ports"])
+        iports = ""
+        for port in node["i_ports"]:
+            iports += f'<tr><td align="left" port="i_{port}">⊙ {port}</td></tr>'
+        oports = ""
+        for port in node["o_ports"]:
+            oports += f'<tr><td align="right" port="o_{port}">{port} ⊙</td></tr>'
+
+        # Use HTML-like label for nodes
+        label = f"""<
+        <table border="0" cellborder="1" cellspacing="0">
+            <tr><td colspan="2" bgcolor="#ddd"><b>{node["name"]}</b></td></tr>
+            <tr><td colspan="2">properties</td></tr>
+            <tr><td colspan="2">extensionGroup<br/>{node["extension_group"]}</td></tr>
+            <tr><td>
+                <table border="0" cellspacing="0">{iports}</table>
+            </td>
+            <td>
+                <table border="0" cellspacing="0">{oports}</table>
+            </td>
+        </tr>    
+        </table>>"""
+        graph.node(node["name"], label)
+
+    # Add edges to the graph
+    for connection in connections:
+        for connection_type in connection_types:
+            if connection_type in connection:
+                for data in connection[connection_type]:
+                    for dest in data["dest"]:
+                        graph.edge(
+                            f'{connection["extension"]}:o_{data["name"]}',
+                            f'{dest["extension"]}:i_{data["name"]}',
+                            color=color(data["name"]),
+                            label=connection_type,
+                        )
+
+    # Save the graph to a file
+    print(graph.source)
+    graph.render("graph", format="png")
+    graph.view()
+
+
+# Load the JSON data
+with open("../property.json") as f:
+    data = json.load(f)
+
+# Create the graph
+create_graph(data)

agents/scripts/install_deps_and_build.sh

@@ -0,0 +1,145 @@
+#!/usr/bin/env bash
+
+# mac, linux
+OS="linux"
+
+# x64, arm64
+CPU="x64"
+
+# debug, release
+BUILD_TYPE="release"
+
+PIP_INSTALL_CMD=${PIP_INSTALL_CMD:-"uv pip install --system"}
+
+build_cxx_extensions() {
+  local app_dir=$1
+
+  if [[ ! -f $app_dir/scripts/BUILD.gn ]]; then
+    echo "FATAL: the scripts/BUILD.gn is required to build cxx extensions."
+    exit 1
+  fi
+
+  cp $app_dir/scripts/BUILD.gn $app_dir
+
+  tgn gen $OS $CPU $BUILD_TYPE -- is_clang=false enable_sanitizer=false
+  tgn build $OS $CPU $BUILD_TYPE
+
+  local ret=$?
+
+  cd $app_dir
+
+  if [[ $ret -ne 0 ]]; then
+    echo "FATAL: failed to build cxx extensions, see logs for detail."
+    exit 1
+  fi
+
+  # Copy the output of ten_packages to the ten_packages/extension/xx/lib.
+  local out="out/$OS/$CPU"
+  for extension in $out/ten_packages/extension/*; do
+    local extension_name=$(basename $extension)
+    if [[ $extension_name == "*" ]]; then
+      echo "No cxx extension, nothing to copy."
+      break
+    fi
+    if [[ ! -d $extension/lib ]]; then
+      echo "No output for extension $extension_name."
+      continue
+    fi
+
+    mkdir -p $app_dir/ten_packages/extension/$extension_name/lib
+    cp -r $extension/lib/* $app_dir/ten_packages/extension/$extension_name/lib
+  done
+}
+
+install_python_requirements() {
+  local app_dir=$1
+
+  if [[ -f "requirements.txt" ]]; then
+    ${PIP_INSTALL_CMD} install -r requirements.txt
+  fi
+
+  # traverse the ten_packages/extension directory to find the requirements.txt
+  if [[ -d "ten_packages/extension" ]]; then
+    for extension in ten_packages/extension/*; do
+      if [[ -f "$extension/requirements.txt" ]]; then
+        ${PIP_INSTALL_CMD} -r $extension/requirements.txt
+      fi
+    done
+  fi
+
+  # traverse the ten_packages/system directory to find the requirements.txt
+  if [[ -d "ten_packages/system" ]]; then
+    for extension in ten_packages/system/*; do
+      if [[ -f "$extension/requirements.txt" ]]; then
+        ${PIP_INSTALL_CMD} -r $extension/requirements.txt
+      fi
+    done
+  fi
+
+  # pre-import llama-index as it cloud download additional resources during the first import
+  echo "pre-import python modules..."
+  python3.10 -c "import llama_index.core;"
+}
+
+build_go_app() {
+  local app_dir=$1
+  cd $app_dir
+
+  go run ten_packages/system/ten_runtime_go/tools/build/main.go --verbose
+  if [[ $? -ne 0 ]]; then
+    echo "FATAL: failed to build go app, see logs for detail."
+    exit 1
+  fi
+}
+
+clean() {
+  local app_dir=$1
+  rm -rf BUILD.gn out
+}
+
+main() {
+  APP_HOME=$(
+    cd $(dirname $0)/..
+    pwd
+  )
+
+  if [[ $1 == "-clean" ]]; then
+    clean $APP_HOME
+    exit 0
+  fi
+
+  if [[ $# -ne 2 ]]; then
+    echo "Usage: $0 <os> <cpu>"
+    exit 1
+  fi
+
+  OS=$1
+  CPU=$2
+
+  echo -e "#include <stdio.h>\n#include <immintrin.h>\nint main() { __m256 a = _mm256_setzero_ps(); return 0; }" > /tmp/test.c
+  if gcc -mavx2 /tmp/test.c -o /tmp/test && ! /tmp/test; then
+    echo "FATAL: unsupported platform."
+    echo "       Please UNCHECK the 'Use Rosetta for x86_64/amd64 emulation on Apple Silicon' Docker Desktop setting if you're running on mac."
+
+    exit 1
+  fi
+
+  if [[ ! -f $APP_HOME/manifest.json ]]; then
+    echo "FATAL: manifest.json is required."
+    exit 1
+  fi
+
+  # Install all dependencies specified in manifest.json.
+  echo "install dependencies..."
+  tman install
+
+  # build extensions and app
+  echo "build_cxx_extensions..."
+  build_cxx_extensions $APP_HOME
+  echo "build_go_app..."
+  build_go_app $APP_HOME
+  echo "install_python_requirements..."
+  install_python_requirements $APP_HOME
+}
+
+main "$@"

agents/scripts/package.sh

@@ -0,0 +1,68 @@
+#!/usr/bin/env bash
+
+APP_HOME=$(
+    cd $(dirname $0)/..
+    pwd
+)
+
+cd $APP_HOME
+
+rm -rf .release
+mkdir .release
+
+copy_package() {
+    local package_type=$1
+    local package_name=$2
+    mkdir -p .release/ten_packages/${package_type}/${package_name}
+
+    if [[ -d ten_packages/${package_type}/${package_name}/lib ]]; then
+        cp -r ten_packages/${package_type}/${package_name}/lib .release/ten_packages/${package_type}/${package_name}/
+    fi
+
+    if [[ -d ten_packages/${package_type}/${package_name}/interface ]]; then
+        cp -r ten_packages/${package_type}/${package_name}/interface .release/ten_packages/${package_type}/${package_name}/
+    fi
+
+    if [[ -f ten_packages/${package_type}/${package_name}/manifest.json ]]; then
+        cp ten_packages/${package_type}/${package_name}/manifest.json .release/ten_packages/${package_type}/${package_name}/
+    fi
+
+    if [[ -f ten_packages/${package_type}/${package_name}/property.json ]]; then
+        cp ten_packages/${package_type}/${package_name}/property.json .release/ten_packages/${package_type}/${package_name}/
+    fi
+
+    # package .py for python extensions
+    # TODO: package 'publish' contents only
+    cp ten_packages/${package_type}/${package_name}/*.py .release/ten_packages/${package_type}/${package_name}/ | true
+    if [[ -f ten_packages/${package_type}/${package_name}/requirements.txt ]]; then
+        cp ten_packages/${package_type}/${package_name}/requirements.txt .release/ten_packages/${package_type}/${package_name}/
+    fi
+
+    # TODO: copy specific contents
+    if [[ -d ten_packages/${package_type}/${package_name}/pb ]]; then
+        cp -r ten_packages/${package_type}/${package_name}/pb .release/ten_packages/${package_type}/${package_name}/
+    fi
+    if [[ -d ten_packages/${package_type}/${package_name}/src ]]; then
+        cp -r ten_packages/${package_type}/${package_name}/src .release/ten_packages/${package_type}/${package_name}/
+    fi
+    if [[ -d ten_packages/${package_type}/${package_name}/realtime ]]; then
+        cp -r ten_packages/${package_type}/${package_name}/realtime .release/ten_packages/${package_type}/${package_name}/
+    fi
+}
+
+cp -r bin .release
+cp manifest.json .release
+cp property.json .release
+
+# copy packages
+mkdir -p .release/ten_packages
+for package_type in system extension_group extension addon_loader; do
+    for package_path in ten_packages/${package_type}/*; do
+        package_name=$(basename ${package_path})
+        copy_package ${package_type} ${package_name}
+    done
+done
+
+if [[ -f session_control.conf ]]; then
+    cp -r session_control.conf .release/
+fi

agents/scripts/pylint.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+
+
+pylint ./agents/ten_packages/extension/. || pylint-exit --warn-fail --error-fail $?

agents/session_control.conf

@@ -0,0 +1,2 @@
+{
+}

agents/ten_packages/bak/litellm_python/__init__.py

@@ -0,0 +1,6 @@
+from . import litellm_addon
+from .extension import EXTENSION_NAME
+from .log import logger
+
+
+logger.info(f"{EXTENSION_NAME} extension loaded")

agents/ten_packages/bak/litellm_python/extension.py

@@ -0,0 +1 @@
+EXTENSION_NAME = "litellm_python"

agents/ten_packages/bak/litellm_python/litellm.py

@@ -0,0 +1,79 @@
+import litellm
+import random
+from typing import Dict, List, Optional
+
+
+class LiteLLMConfig:
+    def __init__(self,
+                 api_key: str,
+                 base_url: str,
+                 frequency_penalty: float,
+                 max_tokens: int,
+                 model: str,
+                 presence_penalty: float,
+                 prompt: str,
+                 provider: str,
+                 temperature: float,
+                 top_p: float,
+                 seed: Optional[int] = None,):
+        self.api_key = api_key
+        self.base_url = base_url
+        self.frequency_penalty = frequency_penalty
+        self.max_tokens = max_tokens
+        self.model = model
+        self.presence_penalty = presence_penalty
+        self.prompt = prompt
+        self.provider = provider
+        self.seed = seed if seed is not None else random.randint(0, 10000)
+        self.temperature = temperature
+        self.top_p = top_p
+
+    @classmethod
+    def default_config(cls):
+        return cls(
+            api_key="",
+            base_url="",
+            max_tokens=512,
+            model="gpt-4o-mini",
+            frequency_penalty=0.9,
+            presence_penalty=0.9,
+            prompt="You are a voice assistant who talks in a conversational way and can chat with me like my friends. I will speak to you in English or Chinese, and you will answer in the corrected and improved version of my text with the language I use. Don’t talk like a robot, instead I would like you to talk like a real human with emotions. I will use your answer for text-to-speech, so don’t return me any meaningless characters. I want you to be helpful, when I’m asking you for advice, give me precise, practical and useful advice instead of being vague. When giving me a list of options, express the options in a narrative way instead of bullet points.",
+            provider="",
+            seed=random.randint(0, 10000),
+            temperature=0.1,
+            top_p=1.0
+        )
+
+
+class LiteLLM:
+    def __init__(self, config: LiteLLMConfig):
+        self.config = config
+
+    def get_chat_completions_stream(self, messages: List[Dict[str, str]]):
+        kwargs = {
+            "api_key": self.config.api_key,
+            "base_url": self.config.base_url,
+            "custom_llm_provider": self.config.provider,
+            "frequency_penalty": self.config.frequency_penalty,
+            "max_tokens": self.config.max_tokens,
+            "messages": [
+                {
+                    "role": "system",
+                    "content": self.config.prompt,
+                },
+                *messages,
+            ],
+            "model": self.config.model,
+            "presence_penalty": self.config.presence_penalty,
+            "seed": self.config.seed,
+            "stream": True,
+            "temperature": self.config.temperature,
+            "top_p": self.config.top_p,
+        }
+
+        try:
+            response = litellm.completion(**kwargs)
+
+            return response
+        except Exception as e:
+            raise Exception(f"get_chat_completions_stream failed, err: {e}")

agents/ten_packages/bak/litellm_python/litellm_addon.py

@@ -0,0 +1,23 @@
+#
+#
+# Agora Real Time Engagement
+# Created by XinHui Li in 2024.
+# Copyright (c) 2024 Agora IO. All rights reserved.
+#
+#
+from ten import (
+    Addon,
+    register_addon_as_extension,
+    TenEnv,
+)
+from .extension import EXTENSION_NAME
+from .log import logger
+from .litellm_extension import LiteLLMExtension
+
+
+@register_addon_as_extension(EXTENSION_NAME)
+class LiteLLMExtensionAddon(Addon):
+    def on_create_instance(self, ten: TenEnv, addon_name: str, context) -> None:
+        logger.info("on_create_instance")
+
+        ten.on_create_instance_done(LiteLLMExtension(addon_name), context)

agents/ten_packages/bak/litellm_python/litellm_extension.py

@@ -0,0 +1,229 @@
+#
+#
+# Agora Real Time Engagement
+# Created by XinHui Li in 2024.
+# Copyright (c) 2024 Agora IO. All rights reserved.
+#
+#
+from threading import Thread
+from ten import (
+    Extension,
+    TenEnv,
+    Cmd,
+    Data,
+    StatusCode,
+    CmdResult,
+)
+from .litellm import LiteLLM, LiteLLMConfig
+from .log import logger
+from .utils import get_micro_ts, parse_sentence
+
+
+CMD_IN_FLUSH = "flush"
+CMD_OUT_FLUSH = "flush"
+DATA_IN_TEXT_DATA_PROPERTY_TEXT = "text"
+DATA_IN_TEXT_DATA_PROPERTY_IS_FINAL = "is_final"
+DATA_OUT_TEXT_DATA_PROPERTY_TEXT = "text"
+DATA_OUT_TEXT_DATA_PROPERTY_TEXT_END_OF_SEGMENT = "end_of_segment"
+
+PROPERTY_API_KEY = "api_key"  # Required
+PROPERTY_BASE_URL = "base_url"  # Optional
+PROPERTY_FREQUENCY_PENALTY = "frequency_penalty"  # Optional
+PROPERTY_GREETING = "greeting"  # Optional
+PROPERTY_MAX_MEMORY_LENGTH = "max_memory_length"  # Optional
+PROPERTY_MAX_TOKENS = "max_tokens"  # Optional
+PROPERTY_MODEL = "model"  # Optional
+PROPERTY_PRESENCE_PENALTY = "presence_penalty"  # Optional
+PROPERTY_PROMPT = "prompt"  # Optional
+PROPERTY_PROVIDER = "provider"  # Optional
+PROPERTY_TEMPERATURE = "temperature"  # Optional
+PROPERTY_TOP_P = "top_p"  # Optional
+
+
+class LiteLLMExtension(Extension):
+    memory = []
+    max_memory_length = 10
+    outdate_ts = 0
+    litellm = None
+
+    def on_start(self, ten: TenEnv) -> None:
+        logger.info("LiteLLMExtension on_start")
+        # Prepare configuration
+        litellm_config = LiteLLMConfig.default_config()
+
+        for key in [PROPERTY_API_KEY, PROPERTY_GREETING, PROPERTY_MODEL, PROPERTY_PROMPT]:
+            try:
+                val = ten.get_property_string(key)
+                if val:
+                    litellm_config.key = val
+            except Exception as e:
+                logger.warning(f"get_property_string optional {key} failed, err: {e}")
+
+        for key in [PROPERTY_FREQUENCY_PENALTY, PROPERTY_PRESENCE_PENALTY, PROPERTY_TEMPERATURE, PROPERTY_TOP_P]:
+            try:
+                litellm_config.key = float(ten.get_property_float(key))
+            except Exception as e:
+                logger.warning(f"get_property_float optional {key} failed, err: {e}")
+
+        for key in [PROPERTY_MAX_MEMORY_LENGTH, PROPERTY_MAX_TOKENS]:
+            try:
+                litellm_config.key = int(ten.get_property_int(key))
+            except Exception as e:
+                logger.warning(f"get_property_int optional {key} failed, err: {e}")
+
+        # Create LiteLLM instance
+        self.litellm = LiteLLM(litellm_config)
+        logger.info(f"newLiteLLM succeed with max_tokens: {litellm_config.max_tokens}, model: {litellm_config.model}")
+
+        # Send greeting if available
+        greeting = ten.get_property_string(PROPERTY_GREETING)
+        if greeting:
+            try:
+                output_data = Data.create("text_data")
+                output_data.set_property_string(DATA_OUT_TEXT_DATA_PROPERTY_TEXT, greeting)
+                output_data.set_property_bool(DATA_OUT_TEXT_DATA_PROPERTY_TEXT_END_OF_SEGMENT, True)
+                ten.send_data(output_data)
+                logger.info(f"greeting [{greeting}] sent")
+            except Exception as e:
+                logger.error(f"greeting [{greeting}] send failed, err: {e}")
+
+        ten.on_start_done()
+
+    def on_stop(self, ten: TenEnv) -> None:
+        logger.info("LiteLLMExtension on_stop")
+        ten.on_stop_done()
+
+    def on_cmd(self, ten: TenEnv, cmd: Cmd) -> None:
+        logger.info("LiteLLMExtension on_cmd")
+        cmd_json = cmd.to_json()
+        logger.info(f"LiteLLMExtension on_cmd json: {cmd_json}")
+
+        cmd_name = cmd.get_name()
+
+        if cmd_name == CMD_IN_FLUSH:
+            self.outdate_ts = get_micro_ts()
+            cmd_out = Cmd.create(CMD_OUT_FLUSH)
+            ten.send_cmd(cmd_out, None)
+            logger.info(f"LiteLLMExtension on_cmd sent flush")
+        else:
+            logger.info(f"LiteLLMExtension on_cmd unknown cmd: {cmd_name}")
+            cmd_result = CmdResult.create(StatusCode.ERROR)
+            cmd_result.set_property_string("detail", "unknown cmd")
+            ten.return_result(cmd_result, cmd)
+            return
+
+        cmd_result = CmdResult.create(StatusCode.OK)
+        cmd_result.set_property_string("detail", "success")
+        ten.return_result(cmd_result, cmd)
+
+    def on_data(self, ten: TenEnv, data: Data) -> None:
+        """
+        on_data receives data from ten graph.
+        current suppotend data:
+          - name: text_data
+            example:
+            {name: text_data, properties: {text: "hello"}
+        """
+        logger.info(f"LiteLLMExtension on_data")
+
+        # Assume 'data' is an object from which we can get properties
+        try:
+            is_final = data.get_property_bool(DATA_IN_TEXT_DATA_PROPERTY_IS_FINAL)
+            if not is_final:
+                logger.info("ignore non-final input")
+                return
+        except Exception as e:
+            logger.error(f"on_data get_property_bool {DATA_IN_TEXT_DATA_PROPERTY_IS_FINAL} failed, err: {e}")
+            return
+
+        # Get input text
+        try:
+            input_text = data.get_property_string(DATA_IN_TEXT_DATA_PROPERTY_TEXT)
+            if not input_text:
+                logger.info("ignore empty text")
+                return
+            logger.info(f"on_data input text: [{input_text}]")
+        except Exception as e:
+            logger.error(f"on_data get_property_string {DATA_IN_TEXT_DATA_PROPERTY_TEXT} failed, err: {e}")
+            return
+
+        # Prepare memory
+        if len(self.memory) > self.max_memory_length:
+            self.memory.pop(0)
+        self.memory.append({"role": "user", "content": input_text})
+
+        def chat_completions_stream_worker(start_time, input_text, memory):
+            try:
+                logger.info(f"chat_completions_stream_worker for input text: [{input_text}] memory: {memory}")
+
+                # Get result from AI
+                resp = self.litellm.get_chat_completions_stream(memory)
+                if resp is None:
+                    logger.info(f"chat_completions_stream_worker for input text: [{input_text}] failed")
+                    return
+
+                sentence = ""
+                full_content = ""
+                first_sentence_sent = False
+
+                for chat_completions in resp:
+                    if start_time < self.outdate_ts:
+                        logger.info(f"chat_completions_stream_worker recv interrupt and flushing for input text: [{input_text}], startTs: {start_time}, outdateTs: {self.outdate_ts}")
+                        break
+
+                    if (len(chat_completions.choices) > 0 and chat_completions.choices[0].delta.content is not None):
+                        content = chat_completions.choices[0].delta.content
+                    else:
+                        content = ""
+
+                    full_content += content
+
+                    while True:
+                        sentence, content, sentence_is_final = parse_sentence(sentence, content)
+
+                        if len(sentence) == 0 or not sentence_is_final:
+                            logger.info(f"sentence {sentence} is empty or not final")
+                            break
+
+                        logger.info(f"chat_completions_stream_worker recv for input text: [{input_text}] got sentence: [{sentence}]")
+
+                        # send sentence
+                        try:
+                            output_data = Data.create("text_data")
+                            output_data.set_property_string(DATA_OUT_TEXT_DATA_PROPERTY_TEXT, sentence)
+                            output_data.set_property_bool(DATA_OUT_TEXT_DATA_PROPERTY_TEXT_END_OF_SEGMENT, False)
+                            ten.send_data(output_data)
+                            logger.info(f"chat_completions_stream_worker recv for input text: [{input_text}] sent sentence [{sentence}]")
+                        except Exception as e:
+                            logger.error(f"chat_completions_stream_worker recv for input text: [{input_text}] send sentence [{sentence}] failed, err: {e}")
+                            break
+
+                        sentence = ""
+                        if not first_sentence_sent:
+                            first_sentence_sent = True
+                            logger.info(f"chat_completions_stream_worker recv for input text: [{input_text}] first sentence sent, first_sentence_latency {get_micro_ts() - start_time}ms")
+
+                # remember response as assistant content in memory
+                memory.append({"role": "assistant", "content": full_content})
+
+                # send end of segment
+                try:
+                    output_data = Data.create("text_data")
+                    output_data.set_property_string(DATA_OUT_TEXT_DATA_PROPERTY_TEXT, sentence)
+                    output_data.set_property_bool(DATA_OUT_TEXT_DATA_PROPERTY_TEXT_END_OF_SEGMENT, True)
+                    ten.send_data(output_data)
+                    logger.info(f"chat_completions_stream_worker for input text: [{input_text}] end of segment with sentence [{sentence}] sent")
+                except Exception as e:
+                    logger.error(f"chat_completions_stream_worker for input text: [{input_text}] end of segment with sentence [{sentence}] send failed, err: {e}")
+
+            except Exception as e:
+                logger.error(f"chat_completions_stream_worker for input text: [{input_text}] failed, err: {e}")
+
+        # Start thread to request and read responses from LiteLLM
+        start_time = get_micro_ts()
+        thread = Thread(
+            target=chat_completions_stream_worker,
+            args=(start_time, input_text, self.memory),
+        )
+        thread.start()
+        logger.info(f"LiteLLMExtension on_data end")

agents/ten_packages/bak/litellm_python/log.py

@@ -0,0 +1,12 @@
+import logging
+from .extension import EXTENSION_NAME
+
+logger = logging.getLogger(EXTENSION_NAME)
+logger.setLevel(logging.INFO)
+
+formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(process)d - [%(filename)s:%(lineno)d] - %(message)s")
+
+console_handler = logging.StreamHandler()
+console_handler.setFormatter(formatter)
+
+logger.addHandler(console_handler)

agents/ten_packages/bak/litellm_python/manifest.json

@@ -0,0 +1,82 @@
+{
+    "type": "extension",
+    "name": "litellm_python",
+    "version": "0.1.0",
+    "dependencies": [
+        {
+            "type": "system",
+            "name": "ten_runtime_python",
+            "version": "0.8"
+        }
+    ],
+    "api": {
+        "property": {
+            "api_key": {
+                "type": "string"
+            },
+            "base_url": {
+                "type": "string"
+            },
+            "frequency_penalty": {
+                "type": "float64"
+            },
+            "greeting": {
+                "type": "string"
+            },
+            "max_memory_length": {
+                "type": "int64"
+            },
+            "max_tokens": {
+                "type": "int64"
+            },
+            "model": {
+                "type": "string"
+            },
+            "presence_penalty": {
+                "type": "float64"
+            },
+            "prompt": {
+                "type": "string"
+            },
+            "provider": {
+                "type": "string"
+            },
+            "temperature": {
+                "type": "float64"
+            },
+            "top_p": {
+                "type": "float64"
+            }
+        },
+        "data_in": [
+            {
+                "name": "text_data",
+                "property": {
+                    "text": {
+                        "type": "string"
+                    }
+                }
+            }
+        ],
+        "data_out": [
+            {
+                "name": "text_data",
+                "property": {
+                    "text": {
+                        "type": "string"
+                    }
+                }
+            }
+        ],
+        "cmd_in": [
+            {
+                "name": "flush"
+            }
+        ],
+        "cmd_out": [
+            {
+                "name": "flush"
+            }
+        ]
+    }
+}

agents/ten_packages/bak/litellm_python/requirements.txt

@@ -0,0 +1 @@
+litellm==1.42.12

agents/ten_packages/bak/litellm_python/utils.py

@@ -0,0 +1,19 @@
+import time
+
+
+def get_micro_ts():
+    return int(time.time() * 1_000_000)
+
+
+def is_punctuation(char: str):
+    return char in [",", "，", ".", "。", "?", "？", "!", "！"]
+
+
+def parse_sentence(sentence: str, content: str):
+    for i, char in enumerate(content):
+        sentence += char
+
+        if is_punctuation(char):
+            return sentence, content[i + 1:], True
+
+    return sentence, "", False

agents/ten_packages/extension/agora_rtm_wrapper/extension.go

@@ -0,0 +1,180 @@
+/**
+ *
+ * Agora Real Time Engagement
+ * Created by Wei Hu in 2022-10.
+ * Copyright (c) 2024 Agora IO. All rights reserved.
+ *
+ */
+// Note that this is just an example extension written in the GO programming
+// language, so the package name does not equal to the containing directory
+// name. However, it is not common in Go.
+package extension
+
+import (
+	"encoding/json"
+	"fmt"
+	"strconv"
+
+	"ten_framework/ten"
+)
+
+// Message colllector represents the text output result
+// @Description 输出结果
+type ColllectorMessage struct {
+	Text     string `json:"text"`      // 识别出的文本
+	IsFinal  bool   `json:"is_final"`  // 是否为最终结果
+	StreamID int32  `json:"stream_id"` // 流ID
+	Type     string `json:"data_type"` // 数据类型
+	Ts       uint64 `json:"text_ts"`   // 时间戳
+}
+
+// Message represents the text output result
+// @Description 输出结果
+type Message struct {
+	Text     string `json:"text"`      // 识别出的文本
+	IsFinal  bool   `json:"is_final"`  // 是否为最终结果
+	StreamID string `json:"stream_id"` // 流ID
+	Type     string `json:"type"`      // 数据类型
+	Ts       uint64 `json:"ts"`        // 时间戳
+}
+
+// RtcUserSate represents the rtc user state
+// @Description RTC用户状态
+type RtcUserSate struct {
+	RemoteUserID string `json:"remote_user_id"` // 远程用户ID
+	State        string `json:"state"`          // 状态
+	Reason       string `json:"reason"`         // 原因
+}
+
+type agoraRtmWrapperExtension struct {
+	ten.DefaultExtension
+}
+
+func newExtension(name string) ten.Extension {
+	return &agoraRtmWrapperExtension{}
+}
+
+// OnData receives data from ten graph.
+func (p *agoraRtmWrapperExtension) OnData(
+	tenEnv ten.TenEnv,
+	data ten.Data,
+) {
+	buf, err := data.GetPropertyBytes("data")
+	if err != nil {
+		tenEnv.LogError("OnData GetProperty data error: " + err.Error())
+		return
+	}
+	tenEnv.LogInfo("AGORA_RTM_WRAPPER_EXTENSION OnData: " + string(buf))
+	colllectorMessage := ColllectorMessage{}
+	err = json.Unmarshal(buf, &colllectorMessage)
+	if err != nil {
+		tenEnv.LogError("OnData Unmarshal data error: " + err.Error())
+		return
+	}
+
+	message := Message{
+		Text:     colllectorMessage.Text,
+		IsFinal:  colllectorMessage.IsFinal,
+		StreamID: strconv.Itoa(int(colllectorMessage.StreamID)),
+		Type:     colllectorMessage.Type,
+		Ts:       colllectorMessage.Ts,
+	}
+	jsonBytes, err := json.Marshal(message)
+	if err != nil {
+		tenEnv.LogError("failed to marshal JSON: " + err.Error())
+		return
+	}
+	tenEnv.LogInfo("AGORA_RTM_WRAPPER_EXTENSION OnData: " + string(jsonBytes))
+
+	cmd, _ := ten.NewCmd("publish")
+
+	err = cmd.SetPropertyBytes("message", jsonBytes)
+	if err != nil {
+		tenEnv.LogError("failed to set property message: " + err.Error())
+		return
+	}
+	if err := tenEnv.SendCmd(cmd, func(_ ten.TenEnv, result ten.CmdResult, _ error) {
+		status, err := result.GetStatusCode()
+		tenEnv.LogInfo(fmt.Sprintf("AGORA_RTM_WRAPPER_EXTENSION publish result %d", status))
+		if status != ten.StatusCodeOk || err != nil {
+			tenEnv.LogError("failed to subscribe")
+		}
+	}); err != nil {
+		tenEnv.LogError("failed to send command " + err.Error())
+	}
+}
+
+func (p *agoraRtmWrapperExtension) OnCmd(tenEnv ten.TenEnv, cmd ten.Cmd) {
+	defer func() {
+		if r := recover(); r != nil {
+			tenEnv.LogError(fmt.Sprintf("OnCmd panic: %v", r))
+		}
+		cmdResult, err := ten.NewCmdResult(ten.StatusCodeOk)
+		if err != nil {
+			tenEnv.LogError(fmt.Sprintf("failed to create cmd result: %v", err))
+			return
+		}
+		tenEnv.ReturnResult(cmdResult, cmd, nil)
+	}()
+	cmdName, err := cmd.GetName()
+	if err != nil {
+		tenEnv.LogError(fmt.Sprintf("failed to get cmd name: %v", err))
+		return
+	}
+	tenEnv.LogInfo(fmt.Sprintf("received command: %s", cmdName))
+	switch cmdName {
+	case "on_user_audio_track_state_changed":
+		// on_user_audio_track_state_changed
+		p.handleUserStateChanged(tenEnv, cmd)
+	default:
+		tenEnv.LogWarn(fmt.Sprintf("unsupported cmd: %s", cmdName))
+	}
+}
+
+func (p *agoraRtmWrapperExtension) handleUserStateChanged(tenEnv ten.TenEnv, cmd ten.Cmd) {
+	remoteUserID, err := cmd.GetPropertyString("remote_user_id")
+	if err != nil {
+		tenEnv.LogError(fmt.Sprintf("failed to get remote_user_id: %v", err))
+		return
+	}
+	state, err := cmd.GetPropertyInt32("state")
+	if err != nil {
+		tenEnv.LogError(fmt.Sprintf("failed to get state: %v", err))
+		return
+	}
+	reason, err := cmd.GetPropertyInt32("reason")
+	if err != nil {
+		tenEnv.LogError(fmt.Sprintf("failed to get reason: %v", err))
+		return
+	}
+	userState := RtcUserSate{
+		RemoteUserID: remoteUserID,
+		State:        strconv.Itoa(int(state)),
+		Reason:       strconv.Itoa(int(reason)),
+	}
+	jsonBytes, err := json.Marshal(userState)
+	if err != nil {
+		tenEnv.LogError("failed to marshal JSON: " + err.Error())
+		return
+	}
+	sendCmd, _ := ten.NewCmd("set_presence_state")
+	sendCmd.SetPropertyString("states", string(jsonBytes))
+	tenEnv.LogInfo("AGORA_RTM_WRAPPER_EXTENSION SetRtmPresenceState " + string(jsonBytes))
+	if err := tenEnv.SendCmd(sendCmd, func(_ ten.TenEnv, result ten.CmdResult, _ error) {
+		status, err := result.GetStatusCode()
+		tenEnv.LogInfo(fmt.Sprintf("AGORA_RTM_WRAPPER_EXTENSION SetRtmPresenceState result %d", status))
+		if status != ten.StatusCodeOk || err != nil {
+			panic("failed to SetRtmPresenceState")
+		}
+	}); err != nil {
+		tenEnv.LogError("failed to send command " + err.Error())
+	}
+}
+
+func init() {
+	// Register addon
+	ten.RegisterAddonAsExtension(
+		"agora_rtm_wrapper",
+		ten.NewDefaultExtensionAddon(newExtension),
+	)
+}

agents/ten_packages/extension/agora_rtm_wrapper/go.mod

@@ -0,0 +1,7 @@
+module agora_rtm_wrapper
+
+go 1.20
+
+replace ten_framework => ../../system/ten_runtime_go/interface
+
+require ten_framework v0.0.0-00010101000000-000000000000

agents/ten_packages/extension/agora_rtm_wrapper/manifest.json

@@ -0,0 +1,32 @@
+{
+  "type": "extension",
+  "name": "agora_rtm_wrapper",
+  "version": "0.1.4",
+  "dependencies": [
+    {
+      "type": "system",
+      "name": "ten_runtime_go",
+      "version": "0.8"
+    }
+  ],
+  "api": {
+    "data_in": [
+      {
+        "name": "data"
+      }
+    ],
+    "cmd_out": [
+      {
+        "name": "publish",
+        "property": {
+          "message": {
+            "type": "buf"
+          }
+        }
+      },
+      {
+        "name": "set_presence_state"
+      }
+    ]
+  }
+}

agents/ten_packages/extension/agora_rtm_wrapper/property.json

@@ -0,0 +1 @@
+{}

agents/ten_packages/extension/aliyun_analyticdb_vector_storage/__init__.py

@@ -0,0 +1 @@
+from . import vector_storage_addon

agents/ten_packages/extension/aliyun_analyticdb_vector_storage/client.py

@@ -0,0 +1,95 @@
+# -*- coding: utf-8 -*-
+
+import asyncio
+import threading
+from typing import Coroutine
+from concurrent.futures import Future
+
+
+from alibabacloud_gpdb20160503.client import Client as gpdb20160503Client
+from alibabacloud_tea_openapi import models as open_api_models
+
+
+# maybe need multiple clients
+class AliGPDBClient:
+    def __init__(self, ten_env, access_key_id, access_key_secret, endpoint):
+        self.stopEvent = asyncio.Event()
+        self.loop = None
+        self.tasks = asyncio.Queue()
+        self.access_key_id = access_key_id
+        self.access_key_secret = access_key_secret
+        self.endpoint = endpoint
+        self.client = self.create_client()
+        self.thread = threading.Thread(
+            target=asyncio.run, args=(self.__thread_routine(),)
+        )
+        self.thread.start()
+        self.ten_env = ten_env
+
+    async def stop_thread(self):
+        self.stopEvent.set()
+
+    def create_client(self) -> gpdb20160503Client:
+        config = open_api_models.Config(
+            access_key_id=self.access_key_id,
+            access_key_secret=self.access_key_secret,
+            endpoint=self.endpoint,
+        )
+        return gpdb20160503Client(config)
+
+    def get(self) -> gpdb20160503Client:
+        return self.client
+
+    def close(self):
+        if (self.loop is not None) and self.thread.is_alive():
+            self.stopEvent.set()
+            asyncio.run_coroutine_threadsafe(self.stop_thread(), self.loop)
+            self.thread.join()
+
+    async def __thread_routine(self):
+        self.ten_env.log_info("client __thread_routine start")
+        self.loop = asyncio.get_running_loop()
+        tasks = set()
+        while not self.stopEvent.is_set():
+            if not self.tasks.empty():
+                coro, future = await self.tasks.get()
+                try:
+                    task = asyncio.create_task(coro)
+                    tasks.add(task)
+                    task.add_done_callback(lambda t: future.set_result(t.result()))
+                except Exception as e:
+                    future.set_exception(e)
+            elif tasks:
+                done, tasks = await asyncio.wait(
+                    tasks, return_when=asyncio.FIRST_COMPLETED
+                )
+                for task in done:
+                    if task.exception():
+                        self.ten_env.log_error(f"task exception: {task.exception()}")
+                        future.set_exception(task.exception())
+            else:
+                await asyncio.sleep(0.1)
+        self.ten_env.log_info("client __thread_routine end")
+
+    async def submit_task(self, coro: Coroutine) -> Future:
+        future = Future()
+        await self.tasks.put((coro, future))
+        return future
+
+    def submit_task_with_new_thread(self, coro: Coroutine) -> Future:
+        future = Future()
+
+        def run_coro_in_new_thread():
+            loop = asyncio.new_event_loop()
+            asyncio.set_event_loop(loop)
+            try:
+                result = loop.run_until_complete(coro)
+                future.set_result(result)
+            except Exception as e:
+                future.set_exception(e)
+            finally:
+                loop.close()
+
+        thread = threading.Thread(target=run_coro_in_new_thread)
+        thread.start()
+        return future

agents/ten_packages/extension/aliyun_analyticdb_vector_storage/manifest.json

@@ -0,0 +1,121 @@
+{
+  "type": "extension",
+  "name": "aliyun_analyticdb_vector_storage",
+  "version": "0.1.0",
+  "dependencies": [
+    {
+      "type": "system",
+      "name": "ten_runtime_python",
+      "version": "0.8"
+    }
+  ],
+  "api": {
+    "property": {
+      "alibaba_cloud_access_key_id": {
+        "type": "string"
+      },
+      "alibaba_cloud_access_key_secret": {
+        "type": "string"
+      },
+      "adbpg_instance_id": {
+        "type": "string"
+      },
+      "adbpg_instance_region": {
+        "type": "string"
+      },
+      "adbpg_account": {
+        "type": "string"
+      },
+      "adbpg_account_password": {
+        "type": "string"
+      },
+      "adbpg_namespace": {
+        "type": "string"
+      },
+      "adbpg_namespace_password": {
+        "type": "string"
+      }
+    },
+    "cmd_in": [
+      {
+        "name": "upsert_vector",
+        "property": {
+          "collection_name": {
+            "type": "string"
+          },
+          "file_name": {
+            "type": "string"
+          },
+          "content": {
+            "type": "string"
+          }
+        }
+      },
+      {
+        "name": "query_vector",
+        "property": {
+          "collection_name": {
+            "type": "string"
+          },
+          "top_k": {
+            "type": "int64"
+          },
+          "embedding": {
+            "type": "array",
+            "items": {
+              "type": "float64"
+            }
+          }
+        },
+        "required": [
+          "collection_name",
+          "top_k",
+          "embedding"
+        ],
+        "result": {
+          "property": {
+            "response": {
+              "type": "array",
+              "items": {
+                "type": "object",
+                "properties": {
+                  "content": {
+                    "type": "string"
+                  },
+                  "score": {
+                    "type": "float64"
+                  }
+                }
+              }
+            }
+          }
+        }
+      },
+      {
+        "name": "create_collection",
+        "property": {
+          "collection_name": {
+            "type": "string"
+          },
+          "dimension": {
+            "type": "int32"
+          }
+        },
+        "required": [
+          "collection_name"
+        ]
+      },
+      {
+        "name": "delete_collection",
+        "property": {
+          "collection_name": {
+            "type": "string"
+          }
+        },
+        "required": [
+          "collection_name"
+        ]
+      }
+    ]
+  }
+}

agents/ten_packages/extension/aliyun_analyticdb_vector_storage/model.py

@@ -0,0 +1,546 @@
+# -*- coding: utf-8 -*-
+
+from alibabacloud_gpdb20160503 import models as gpdb_20160503_models  # type: ignore
+import time
+import json
+from typing import Dict, List, Any, Tuple
+from alibabacloud_tea_util import models as util_models
+
+
+class Model:
+    def __init__(self, ten_env, region_id, dbinstance_id, client):
+        self.region_id = region_id
+        self.dbinstance_id = dbinstance_id
+        self.client = client
+        self.read_timeout = 10 * 1000
+        self.connect_timeout = 10 * 1000
+        self.ten_env = ten_env
+
+    def get_client(self):
+        return self.client.get()
+
+    def init_vector_database(self, account, account_password) -> None:
+        try:
+            request = gpdb_20160503_models.InitVectorDatabaseRequest(
+                region_id=self.region_id,
+                dbinstance_id=self.dbinstance_id,
+                manager_account=account,
+                manager_account_password=account_password,
+            )
+            runtime = util_models.RuntimeOptions(
+                read_timeout=self.read_timeout, connect_timeout=self.connect_timeout
+            )
+            response = self.get_client().init_vector_database_with_options(
+                request, runtime
+            )
+            self.ten_env.log_debug(
+                f"init_vector_database response code: {response.status_code}, body:{response.body}"
+            )
+        except Exception as e:
+            self.ten_env.log_error(f"Error: {e}")
+            return e
+
+    async def init_vector_database_async(self, account, account_password) -> None:
+        try:
+            request = gpdb_20160503_models.InitVectorDatabaseRequest(
+                region_id=self.region_id,
+                dbinstance_id=self.dbinstance_id,
+                manager_account=account,
+                manager_account_password=account_password,
+            )
+            runtime = util_models.RuntimeOptions(
+                read_timeout=self.read_timeout, connect_timeout=self.connect_timeout
+            )
+            response = await self.get_client().init_vector_database_with_options_async(
+                request, runtime
+            )
+            self.ten_env.log_debug(
+                f"init_vector_database response code: {response.status_code}, body:{response.body}"
+            )
+        except Exception as e:
+            self.ten_env.log_error(f"Error: {e}")
+            return e
+
+    def create_namespace(
+        self, account, account_password, namespace, namespace_password
+    ) -> None:
+        try:
+            request = gpdb_20160503_models.CreateNamespaceRequest(
+                region_id=self.region_id,
+                dbinstance_id=self.dbinstance_id,
+                manager_account=account,
+                manager_account_password=account_password,
+                namespace=namespace,
+                namespace_password=namespace_password,
+            )
+            runtime = util_models.RuntimeOptions(
+                read_timeout=self.read_timeout, connect_timeout=self.connect_timeout
+            )
+            response = self.get_client().create_namespace_with_options(request, runtime)
+            self.ten_env.log_debug(
+                f"create_namespace response code: {response.status_code}, body:{response.body}"
+            )
+        except Exception as e:
+            self.ten_env.log_error(f"Error: {e}")
+            return e
+
+    async def create_namespace_async(
+        self, account, account_password, namespace, namespace_password
+    ) -> None:
+        try:
+            request = gpdb_20160503_models.CreateNamespaceRequest(
+                region_id=self.region_id,
+                dbinstance_id=self.dbinstance_id,
+                manager_account=account,
+                manager_account_password=account_password,
+                namespace=namespace,
+                namespace_password=namespace_password,
+            )
+            runtime = util_models.RuntimeOptions(
+                read_timeout=self.read_timeout, connect_timeout=self.connect_timeout
+            )
+            response = await self.get_client().create_namespace_with_options_async(
+                request, runtime
+            )
+            self.ten_env.log_debug(
+                f"create_namespace response code: {response.status_code}, body:{response.body}"
+            )
+        except Exception as e:
+            self.ten_env.log_error(f"Error: {e}")
+            return e
+
+    def create_collection(
+        self,
+        account,
+        account_password,
+        namespace,
+        collection,
+        parser: str = None,
+        metrics: str = None,
+        hnsw_m: int = None,
+        pq_enable: int = None,
+        external_storage: int = None,
+    ) -> None:
+        try:
+            metadata = '{"update_ts": "bigint", "file_name": "text", "content": "text"}'
+            full_text_retrieval_fields = "update_ts,file_name"
+            request = gpdb_20160503_models.CreateCollectionRequest(
+                region_id=self.region_id,
+                dbinstance_id=self.dbinstance_id,
+                manager_account=account,
+                manager_account_password=account_password,
+                namespace=namespace,
+                collection=collection,
+                metadata=metadata,
+                full_text_retrieval_fields=full_text_retrieval_fields,
+                parser=parser,
+                metrics=metrics,
+                hnsw_m=hnsw_m,
+                pq_enable=pq_enable,
+                external_storage=external_storage,
+            )
+            runtime = util_models.RuntimeOptions(
+                read_timeout=self.read_timeout, connect_timeout=self.connect_timeout
+            )
+            response = self.get_client().create_collection_with_options(
+                request, runtime
+            )
+            self.ten_env.log_debug(
+                f"create_document_collection response code: {response.status_code}, body:{response.body}"
+            )
+        except Exception as e:
+            self.ten_env.log_error(f"Error: {e}")
+            return e
+
+    async def create_collection_async(
+        self,
+        account,
+        account_password,
+        namespace,
+        collection,
+        parser: str = None,
+        metrics: str = None,
+        hnsw_m: int = None,
+        pq_enable: int = None,
+        external_storage: int = None,
+    ) -> None:
+        try:
+            metadata = '{"update_ts": "bigint", "file_name": "text", "content": "text"}'
+            full_text_retrieval_fields = "update_ts,file_name"
+            request = gpdb_20160503_models.CreateCollectionRequest(
+                region_id=self.region_id,
+                dbinstance_id=self.dbinstance_id,
+                manager_account=account,
+                manager_account_password=account_password,
+                namespace=namespace,
+                collection=collection,
+                metadata=metadata,
+                full_text_retrieval_fields=full_text_retrieval_fields,
+                parser=parser,
+                metrics=metrics,
+                hnsw_m=hnsw_m,
+                pq_enable=pq_enable,
+                external_storage=external_storage,
+            )
+            runtime = util_models.RuntimeOptions(
+                read_timeout=self.read_timeout, connect_timeout=self.connect_timeout
+            )
+            response = await self.get_client().create_collection_with_options_async(
+                request, runtime
+            )
+            self.ten_env.log_debug(
+                f"create_document_collection response code: {response.status_code}, body:{response.body}"
+            )
+        except Exception as e:
+            self.ten_env.log_error(f"Error: {e}")
+            return e
+
+    def delete_collection(self, namespace, namespace_password, collection) -> None:
+        try:
+            request = gpdb_20160503_models.DeleteCollectionRequest(
+                region_id=self.region_id,
+                dbinstance_id=self.dbinstance_id,
+                namespace_password=namespace_password,
+                namespace=namespace,
+                collection=collection,
+            )
+            runtime = util_models.RuntimeOptions(
+                read_timeout=self.read_timeout, connect_timeout=self.connect_timeout
+            )
+            response = self.get_client().delete_collection_with_options(
+                request, runtime
+            )
+            self.ten_env.log_debug(
+                f"delete_collection response code: {response.status_code}, body:{response.body}"
+            )
+        except Exception as e:
+            self.ten_env.log_error(f"Error: {e}")
+            return e
+
+    async def delete_collection_async(
+        self, namespace, namespace_password, collection
+    ) -> None:
+        try:
+            request = gpdb_20160503_models.DeleteCollectionRequest(
+                region_id=self.region_id,
+                dbinstance_id=self.dbinstance_id,
+                namespace_password=namespace_password,
+                namespace=namespace,
+                collection=collection,
+            )
+            runtime = util_models.RuntimeOptions(
+                read_timeout=self.read_timeout, connect_timeout=self.connect_timeout
+            )
+            response = await self.get_client().delete_collection_with_options_async(
+                request, runtime
+            )
+            self.ten_env.log_info(
+                f"delete_collection response code: {response.status_code}, body:{response.body}"
+            )
+        except Exception as e:
+            self.ten_env.log_error(f"Error: {e}")
+            return e
+
+    def upsert_collection_data(
+        self,
+        collection,
+        namespace,
+        namespace_password,
+        rows: List[Tuple[str, str, List[float]]] = None,
+    ) -> None:
+        try:
+            request_rows = []
+            for row in rows:
+                file_name = row[0]
+                content = row[1]
+                vector = row[2]
+                metadata = {
+                    "update_ts": int(time.time() * 1000),
+                    "file_name": file_name,
+                    "content": content,
+                }
+                request_row = gpdb_20160503_models.UpsertCollectionDataRequestRows(
+                    metadata=metadata, vector=vector
+                )
+                request_rows.append(request_row)
+            upsert_collection_data_request = (
+                gpdb_20160503_models.UpsertCollectionDataRequest(
+                    region_id=self.region_id,
+                    dbinstance_id=self.dbinstance_id,
+                    collection=collection,
+                    namespace_password=namespace_password,
+                    namespace=namespace,
+                    rows=request_rows,
+                )
+            )
+            runtime = util_models.RuntimeOptions(
+                read_timeout=self.read_timeout, connect_timeout=self.connect_timeout
+            )
+            response = self.get_client().upsert_collection_data_with_options(
+                upsert_collection_data_request, runtime
+            )
+            self.ten_env.log_debug(
+                f"upsert_collection response code: {response.status_code}, body:{response.body}"
+            )
+        except Exception as e:
+            self.ten_env.log_error(f"Error: {e}")
+            return e
+
+    async def upsert_collection_data_async(
+        self,
+        collection,
+        namespace,
+        namespace_password,
+        rows: List[Tuple[str, str, List[float]]] = None,
+    ) -> None:
+        try:
+            request_rows = []
+            for row in rows:
+                file_name = row[0]
+                content = row[1]
+                vector = row[2]
+                metadata = {
+                    "update_ts": int(time.time() * 1000),
+                    "file_name": file_name,
+                    "content": content,
+                }
+                request_row = gpdb_20160503_models.UpsertCollectionDataRequestRows(
+                    metadata=metadata, vector=vector
+                )
+                request_rows.append(request_row)
+            upsert_collection_data_request = (
+                gpdb_20160503_models.UpsertCollectionDataRequest(
+                    region_id=self.region_id,
+                    dbinstance_id=self.dbinstance_id,
+                    collection=collection,
+                    namespace_password=namespace_password,
+                    namespace=namespace,
+                    rows=request_rows,
+                )
+            )
+            runtime = util_models.RuntimeOptions(
+                read_timeout=self.read_timeout, connect_timeout=self.connect_timeout
+            )
+            response = (
+                await self.get_client().upsert_collection_data_with_options_async(
+                    upsert_collection_data_request, runtime
+                )
+            )
+            self.ten_env.log_debug(
+                f"upsert_collection response code: {response.status_code}, body:{response.body}"
+            )
+        except Exception as e:
+            self.ten_env.log_error(f"Error: {e}")
+            return e
+
+    # pylint: disable=redefined-builtin
+    def query_collection_data(
+        self,
+        collection,
+        namespace,
+        namespace_password,
+        vector: List[float] = None,
+        top_k: int = 10,
+        content: str = None,
+        filter: str = None,
+        hybrid_search: str = None,
+        hybrid_search_args: Dict[str, dict] = None,
+        include_metadata_fields: str = None,
+        include_values: bool = None,
+        metrics: str = None,
+    ) -> Tuple[Any, Any]:
+        try:
+            query_collection_data_request = (
+                gpdb_20160503_models.QueryCollectionDataRequest(
+                    region_id=self.region_id,
+                    dbinstance_id=self.dbinstance_id,
+                    collection=collection,
+                    namespace_password=namespace_password,
+                    namespace=namespace,
+                    vector=vector,
+                    top_k=top_k,
+                    content=content,
+                    filter=filter,
+                    hybrid_search=hybrid_search,
+                    hybrid_search_args=hybrid_search_args,
+                    include_metadata_fields=include_metadata_fields,
+                    include_values=include_values,
+                    metrics=metrics,
+                )
+            )
+            runtime = util_models.RuntimeOptions(
+                read_timeout=self.read_timeout, connect_timeout=self.connect_timeout
+            )
+            response = self.get_client().query_collection_data_with_options(
+                query_collection_data_request, runtime
+            )
+            self.ten_env.log_debug(f"query_collection response code: {response.status_code}")
+            return response, None
+        except Exception as e:
+            self.ten_env.log_error(f"Error: {e}")
+            return None, e
+
+    # pylint: disable=redefined-builtin
+    async def query_collection_data_async(
+        self,
+        collection,
+        namespace,
+        namespace_password,
+        vector: List[float] = None,
+        top_k: int = 10,
+        content: str = None,
+        filter: str = None,
+        hybrid_search: str = None,
+        hybrid_search_args: Dict[str, dict] = None,
+        include_metadata_fields: str = None,
+        include_values: bool = None,
+        metrics: str = None,
+    ) -> Tuple[Any, Any]:
+        try:
+            query_collection_data_request = (
+                gpdb_20160503_models.QueryCollectionDataRequest(
+                    region_id=self.region_id,
+                    dbinstance_id=self.dbinstance_id,
+                    collection=collection,
+                    namespace_password=namespace_password,
+                    namespace=namespace,
+                    vector=vector,
+                    top_k=top_k,
+                    content=content,
+                    filter=filter,
+                    hybrid_search=hybrid_search,
+                    hybrid_search_args=hybrid_search_args,
+                    include_metadata_fields=include_metadata_fields,
+                    include_values=include_values,
+                    metrics=metrics,
+                )
+            )
+            runtime = util_models.RuntimeOptions(
+                read_timeout=self.read_timeout, connect_timeout=self.connect_timeout
+            )
+            response = await self.get_client().query_collection_data_with_options_async(
+                query_collection_data_request, runtime
+            )
+            self.ten_env.log_debug(f"query_collection response code: {response.status_code}")
+            return response, None
+        except Exception as e:
+            self.ten_env.log_error(f"Error: {e}")
+            return None, e
+
+    def parse_collection_data(
+        self, body: gpdb_20160503_models.QueryCollectionDataResponseBody
+    ) -> str:
+        try:
+            matches = body.to_map()["Matches"]["match"]
+            results = [
+                {"content": match["Metadata"]["content"], "score": match["Score"]}
+                for match in matches
+            ]
+            results.sort(key=lambda x: x["score"], reverse=True)
+            json_str = json.dumps(results)
+            return json_str
+        except Exception as e:
+            self.ten_env.log_error(
+                f"parse collection data failed, error: {e}, data: {body.to_map()}"
+            )
+            return "[]"
+
+    def list_collections(self, namespace, namespace_password) -> Tuple[List[str], Any]:
+        try:
+            request = gpdb_20160503_models.ListCollectionsRequest(
+                region_id=self.region_id,
+                dbinstance_id=self.dbinstance_id,
+                namespace=namespace,
+                namespace_password=namespace_password,
+            )
+            runtime = util_models.RuntimeOptions(
+                read_timeout=self.read_timeout, connect_timeout=self.connect_timeout
+            )
+            response = self.get_client().list_collections_with_options(request, runtime)
+            self.ten_env.log_debug(
+                f"list_collections response code: {response.status_code}, body:{response.body}"
+            )
+            collections = response.body.to_map()["Collections"]["collection"]
+            return collections, None
+        except Exception as e:
+            self.ten_env.log_error(f"Error: {e}")
+            return [], e
+
+    async def list_collections_async(
+        self, namespace, namespace_password
+    ) -> Tuple[List[str], Any]:
+        try:
+            request = gpdb_20160503_models.ListCollectionsRequest(
+                region_id=self.region_id,
+                dbinstance_id=self.dbinstance_id,
+                namespace=namespace,
+                namespace_password=namespace_password,
+            )
+            runtime = util_models.RuntimeOptions(
+                read_timeout=self.read_timeout, connect_timeout=self.connect_timeout
+            )
+            response = await self.get_client().list_collections_with_options_async(
+                request, runtime
+            )
+            self.ten_env.log_debug(
+                f"list_collections response code: {response.status_code}, body:{response.body}"
+            )
+            collections = response.body.to_map()["Collections"]["collection"]
+            return collections, None
+        except Exception as e:
+            self.ten_env.log_error(f"Error: {e}")
+            return [], e
+
+    def create_vector_index(
+        self, account, account_password, namespace, collection, dimension
+    ) -> None:
+        try:
+            request = gpdb_20160503_models.CreateVectorIndexRequest(
+                region_id=self.region_id,
+                dbinstance_id=self.dbinstance_id,
+                manager_account=account,
+                manager_account_password=account_password,
+                namespace=namespace,
+                collection=collection,
+                dimension=dimension,
+                pq_enable=0,
+            )
+            runtime = util_models.RuntimeOptions(
+                read_timeout=self.read_timeout, connect_timeout=self.connect_timeout
+            )
+            response = self.get_client().create_vector_index_with_options(
+                request, runtime
+            )
+            self.ten_env.log_debug(
+                f"create_vector_index response code: {response.status_code}, body:{response.body}"
+            )
+        except Exception as e:
+            self.ten_env.log_error(f"Error: {e}")
+            return e
+
+    async def create_vector_index_async(
+        self, account, account_password, namespace, collection, dimension
+    ) -> None:
+        try:
+            request = gpdb_20160503_models.CreateVectorIndexRequest(
+                region_id=self.region_id,
+                dbinstance_id=self.dbinstance_id,
+                manager_account=account,
+                manager_account_password=account_password,
+                namespace=namespace,
+                collection=collection,
+                dimension=dimension,
+                pq_enable=0,
+            )
+            runtime = util_models.RuntimeOptions(
+                read_timeout=self.read_timeout, connect_timeout=self.connect_timeout
+            )
+            response = await self.get_client().create_vector_index_with_options_async(
+                request, runtime
+            )
+            self.ten_env.log_debug(
+                f"create_vector_index response code: {response.status_code}, body:{response.body}"
+            )
+        except Exception as e:
+            self.ten_env.log_error(f"Error: {e}")
+            return e