m5/StackFlow
0
0
Fork 0
mirror of https://github.com/m5stack/StackFlow.git synced 2026-05-20 11:32:11 -07:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #4 from m5stack/dev

Browse Source 
Dev
This commit is contained in:
dianjixz
2025-01-21 10:09:29 +08:00
committed by GitHub
parent 9d1dfced01 a830d441bf
commit c490c4dfd2
690 changed files with 182687 additions and 879 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitignore
+2 -1
View File
@@ -1,3 +1,4 @@
.vscode/settings.json
projects/core135_llm_product_test_ui
projects/imx678_test
projects/imx678_test
projects/test_*
ext_components/ax-samples/SConstruct
+2 -3
View File
@@ -29,14 +29,13 @@ if "CONFIG_AX_SAMPLES_ENABLED" in os.environ:
LINK_SEARCH_PATH = []
INCLUDE += [
os.path.join(env["GIT_REPO_LISTS"]["ax-samples"]["path"], "examples/base"),
os.path.join(env["GIT_REPO_LISTS"]["ax-samples"]["path"], "examples/utilities"),
os.path.join(env["GIT_REPO_LISTS"]["ax-samples"]["path"], "examples"),
]
if "CONFIG_AX_620E_MSP_ENABLED" in os.environ:
INCLUDE += [
os.path.join(
env["GIT_REPO_LISTS"]["ax-samples"]["path"],
"examples/ax620e/middleware",
"examples/ax620e",
)
]
projects/llm_framework/SConstruct
+3 -1
View File
@@ -5,7 +5,7 @@ import shutil
os.environ['SDK_PATH'] = os.path.normpath(str(Path(os.getcwd())/'..'/'..'/'SDK'))
os.environ['EXT_COMPONENTS_PATH'] = os.path.normpath(str(Path(os.getcwd())/'..'/'..'/'ext_components'))
version = 'v0.0.5'
version = 'v0.0.7'
static_lib = 'static_lib'
update = False
@@ -26,4 +26,6 @@ if update:
exec(f.read())
down_url = "https://m5stack.oss-cn-shenzhen.aliyuncs.com/resource/linux/llm/static_lib_{}.tar.gz".format(version)
down_path = check_wget_down(down_url, "static_lib_{}.tar.gz".format(version))
if os.path.exists(static_lib):
shutil.rmtree(static_lib)
shutil.move(down_path, static_lib)
projects/llm_framework/include/sherpa/sherpa-onnx/csrc/fast-clustering-config.h
+39
View File
@@ -0,0 +1,39 @@
// sherpa-onnx/csrc/fast-clustering-config.h
//
// Copyright (c) 2024 Xiaomi Corporation
#ifndef SHERPA_ONNX_CSRC_FAST_CLUSTERING_CONFIG_H_
#define SHERPA_ONNX_CSRC_FAST_CLUSTERING_CONFIG_H_
#include <string>
#include "sherpa-onnx/csrc/parse-options.h"
namespace sherpa_onnx {
struct FastClusteringConfig {
// If greater than 0, then threshold is ignored.
//
// We strongly recommend that you set it if you know the number of clusters
// in advance
int32_t num_clusters = -1;
// distance threshold.
//
// The smaller, the more clusters it will generate.
// The larger, the fewer clusters it will generate.
float threshold = 0.5;
FastClusteringConfig() = default;
FastClusteringConfig(int32_t num_clusters, float threshold)
: num_clusters(num_clusters), threshold(threshold) {}
std::string ToString() const;
void Register(ParseOptions *po);
bool Validate() const;
};
} // namespace sherpa_onnx
#endif // SHERPA_ONNX_CSRC_FAST_CLUSTERING_CONFIG_H_
projects/llm_framework/include/sherpa/sherpa-onnx/csrc/fast-clustering.h
+43
View File
@@ -0,0 +1,43 @@
// sherpa-onnx/csrc/fast-clustering.h
//
// Copyright (c) 2024 Xiaomi Corporation
#ifndef SHERPA_ONNX_CSRC_FAST_CLUSTERING_H_
#define SHERPA_ONNX_CSRC_FAST_CLUSTERING_H_
#include <memory>
#include <vector>
#include "sherpa-onnx/csrc/fast-clustering-config.h"
namespace sherpa_onnx {
class FastClustering {
public:
explicit FastClustering(const FastClusteringConfig &config);
~FastClustering();
/**
* @param features Pointer to a 2-D feature matrix in row major. Each row
* is a feature frame. It is changed in-place. We will
* convert each feature frame to a normalized vector.
* That is, the L2-norm of each vector will be equal to 1.
* It uses cosine dissimilarity,
* which is 1 - (cosine similarity)
* @param num_rows Number of feature frames
* @param num-cols The feature dimension.
*
* @return Return a vector of size num_rows. ans[i] contains the label
* for the i-th feature frame, i.e., the i-th row of the feature
* matrix.
*/
std::vector<int32_t> Cluster(float *features, int32_t num_rows,
int32_t num_cols) const;
private:
class Impl;
std::unique_ptr<Impl> impl_;
};
} // namespace sherpa_onnx
#endif // SHERPA_ONNX_CSRC_FAST_CLUSTERING_H_
projects/llm_framework/include/sherpa/sherpa-onnx/csrc/features.h
+1
View File
@@ -57,6 +57,7 @@ struct FeatureExtractorConfig {
float frame_length_ms = 25.0f; // in milliseconds.
bool is_librosa = false;
bool remove_dc_offset = true; // Subtract mean of wave before FFT.
float preemph_coeff = 0.97f; // Preemphasis coefficient.
std::string window_type = "povey"; // e.g. Hamming window
// For models from NeMo
projects/llm_framework/include/sherpa/sherpa-onnx/csrc/fst-utils.h
+18
View File
@@ -0,0 +1,18 @@
// sherpa-onnx/csrc/fst-utils.h
//
// Copyright (c) 2024 Xiaomi Corporation
#ifndef SHERPA_ONNX_CSRC_FST_UTILS_H_
#define SHERPA_ONNX_CSRC_FST_UTILS_H_
#include <string>
#include "fst/fstlib.h"
namespace sherpa_onnx {
fst::Fst<fst::StdArc> *ReadGraph(const std::string &filename);
}
#endif // SHERPA_ONNX_CSRC_FST_UTILS_H_
projects/llm_framework/include/sherpa/sherpa-onnx/csrc/hypothesis.h
+6 -1
View File
@@ -51,8 +51,13 @@ struct Hypothesis {
// LM log prob if any.
double lm_log_prob = 0;
// the nn lm score for next token given the current ys
// the nn lm score for next token given the current ys,
// when using shallow fusion
CopyableOrtValue nn_lm_scores;
// cur scored tokens by RNN LM, when rescoring
int32_t cur_scored_pos = 0;
// the nn lm states
std::vector<CopyableOrtValue> nn_lm_states;
projects/llm_framework/include/sherpa/sherpa-onnx/csrc/keyword-spotter-transducer-impl.h
+19 -3
View File
@@ -66,15 +66,25 @@ class KeywordSpotterTransducerImpl : public KeywordSpotterImpl {
public:
explicit KeywordSpotterTransducerImpl(const KeywordSpotterConfig &config)
: config_(config),
model_(OnlineTransducerModel::Create(config.model_config)),
sym_(config.model_config.tokens) {
model_(OnlineTransducerModel::Create(config.model_config)) {
if (!config.model_config.tokens_buf.empty()) {
sym_ = SymbolTable(config.model_config.tokens_buf, false);
} else {
/// assuming tokens_buf and tokens are guaranteed not being both empty
sym_ = SymbolTable(config.model_config.tokens, true);
}
if (sym_.Contains("<unk>")) {
unk_id_ = sym_["<unk>"];
}
model_->SetFeatureDim(config.feat_config.feature_dim);
InitKeywords();
if (config.keywords_buf.empty()) {
InitKeywords();
} else {
InitKeywordsFromBufStr();
}
decoder_ = std::make_unique<TransducerKeywordDecoder>(
model_.get(), config_.max_active_paths, config_.num_trailing_blanks,
@@ -305,6 +315,12 @@ class KeywordSpotterTransducerImpl : public KeywordSpotterImpl {
}
#endif
void InitKeywordsFromBufStr() {
// keywords_buf's content is supposed to be same as the keywords_file's
std::istringstream is(config_.keywords_buf);
InitKeywords(is);
}
void InitOnlineStream(OnlineStream *stream) const {
auto r = decoder_->GetEmptyResult();
SHERPA_ONNX_CHECK_EQ(r.hyps.Size(), 1);
projects/llm_framework/include/sherpa/sherpa-onnx/csrc/keyword-spotter.h
+5
View File
@@ -69,6 +69,11 @@ struct KeywordSpotterConfig {
std::string keywords_file;
/// if keywords_buf is non-empty,
/// the keywords will be loaded from the buffer instead of from the
/// "keywrods_file"
std::string keywords_buf;
KeywordSpotterConfig() = default;
KeywordSpotterConfig(const FeatureExtractorConfig &feat_config,
projects/llm_framework/include/sherpa/sherpa-onnx/csrc/lexicon.h
+5 -10
View File
@@ -6,17 +6,13 @@
#define SHERPA_ONNX_CSRC_LEXICON_H_
#include <cstdint>
#include <istream>
#include <memory>
#include <string>
#include <unordered_map>
#include <unordered_set>
#include <vector>
#if __ANDROID_API__ >= 9
#include "android/asset_manager.h"
#include "android/asset_manager_jni.h"
#endif
#include "sherpa-onnx/csrc/offline-tts-frontend.h"
namespace sherpa_onnx {
@@ -30,11 +26,10 @@ class Lexicon : public OfflineTtsFrontend {
const std::string &punctuations, const std::string &language,
bool debug = false);
#if __ANDROID_API__ >= 9
Lexicon(AAssetManager *mgr, const std::string &lexicon,
const std::string &tokens, const std::string &punctuations,
const std::string &language, bool debug = false);
#endif
template <typename Manager>
Lexicon(Manager *mgr, const std::string &lexicon, const std::string &tokens,
const std::string &punctuations, const std::string &language,
bool debug = false);
std::vector<TokenIDs> ConvertTextToTokenIds(
const std::string &text, const std::string &voice = "") const override;
projects/llm_framework/include/sherpa/sherpa-onnx/csrc/macros.h
+121 -105
View File
@@ -5,6 +5,19 @@
#ifndef SHERPA_ONNX_CSRC_MACROS_H_
#define SHERPA_ONNX_CSRC_MACROS_H_
#include <stdio.h>
#include <stdlib.h>
#include <utility>
#if __OHOS__
#include "hilog/log.h"
#undef LOG_DOMAIN
#undef LOG_TAG
// https://gitee.com/openharmony/docs/blob/145a084f0b742e4325915e32f8184817927d1251/en/contribute/OpenHarmony-Log-guide.md#hilog-api-usage-specifications
#define LOG_DOMAIN 0x6666
#define LOG_TAG "sherpa_onnx"
#endif
#if __ANDROID_API__ >= 8
#include "android/log.h"
@@ -16,6 +29,8 @@
fprintf(stderr, "\n"); \
__android_log_print(ANDROID_LOG_WARN, "sherpa-onnx", ##__VA_ARGS__); \
} while (0)
#elif defined(__OHOS__)
#define SHERPA_ONNX_LOGE(...) OH_LOG_INFO(LOG_APP, ##__VA_ARGS__)
#elif SHERPA_ONNX_ENABLE_WASM
#define SHERPA_ONNX_LOGE(...) \
do { \
@@ -35,138 +50,139 @@
#endif
// Read an integer
#define SHERPA_ONNX_READ_META_DATA(dst, src_key) \
do { \
auto value = \
meta_data.LookupCustomMetadataMapAllocated(src_key, allocator); \
if (!value) { \
SHERPA_ONNX_LOGE("%s does not exist in the metadata", src_key); \
exit(-1); \
} \
\
dst = atoi(value.get()); \
if (dst < 0) { \
SHERPA_ONNX_LOGE("Invalid value %d for %s", dst, src_key); \
exit(-1); \
} \
#define SHERPA_ONNX_READ_META_DATA(dst, src_key) \
do { \
auto value = LookupCustomModelMetaData(meta_data, src_key, allocator); \
if (value.empty()) { \
SHERPA_ONNX_LOGE("'%s' does not exist in the metadata", src_key); \
exit(-1); \
} \
\
dst = atoi(value.c_str()); \
if (dst < 0) { \
SHERPA_ONNX_LOGE("Invalid value %d for '%s'", dst, src_key); \
exit(-1); \
} \
} while (0)
#define SHERPA_ONNX_READ_META_DATA_WITH_DEFAULT(dst, src_key, default_value) \
do { \
auto value = \
meta_data.LookupCustomMetadataMapAllocated(src_key, allocator); \
if (!value) { \
auto value = LookupCustomModelMetaData(meta_data, src_key, allocator); \
if (value.empty()) { \
dst = default_value; \
} else { \
dst = atoi(value.get()); \
dst = atoi(value.c_str()); \
if (dst < 0) { \
SHERPA_ONNX_LOGE("Invalid value %d for %s", dst, src_key); \
SHERPA_ONNX_LOGE("Invalid value %d for '%s'", dst, src_key); \
exit(-1); \
} \
} \
} while (0)
// read a vector of integers
#define SHERPA_ONNX_READ_META_DATA_VEC(dst, src_key) \
do { \
auto value = \
meta_data.LookupCustomMetadataMapAllocated(src_key, allocator); \
if (!value) { \
SHERPA_ONNX_LOGE("%s does not exist in the metadata", src_key); \
exit(-1); \
} \
\
bool ret = SplitStringToIntegers(value.get(), ",", true, &dst); \
if (!ret) { \
SHERPA_ONNX_LOGE("Invalid value %s for %s", value.get(), src_key); \
exit(-1); \
} \
#define SHERPA_ONNX_READ_META_DATA_VEC(dst, src_key) \
do { \
auto value = LookupCustomModelMetaData(meta_data, src_key, allocator); \
if (value.empty()) { \
SHERPA_ONNX_LOGE("'%s' does not exist in the metadata", src_key); \
exit(-1); \
} \
\
bool ret = SplitStringToIntegers(value.c_str(), ",", true, &dst); \
if (!ret) { \
SHERPA_ONNX_LOGE("Invalid value '%s' for '%s'", value.c_str(), src_key); \
exit(-1); \
} \
} while (0)
// read a vector of floats
#define SHERPA_ONNX_READ_META_DATA_VEC_FLOAT(dst, src_key) \
do { \
auto value = \
meta_data.LookupCustomMetadataMapAllocated(src_key, allocator); \
if (!value) { \
SHERPA_ONNX_LOGE("%s does not exist in the metadata", src_key); \
exit(-1); \
} \
\
bool ret = SplitStringToFloats(value.get(), ",", true, &dst); \
if (!ret) { \
SHERPA_ONNX_LOGE("Invalid value %s for %s", value.get(), src_key); \
exit(-1); \
} \
#define SHERPA_ONNX_READ_META_DATA_VEC_FLOAT(dst, src_key) \
do { \
auto value = LookupCustomModelMetaData(meta_data, src_key, allocator); \
if (value.empty()) { \
SHERPA_ONNX_LOGE("%s does not exist in the metadata", src_key); \
exit(-1); \
} \
\
bool ret = SplitStringToFloats(value.c_str(), ",", true, &dst); \
if (!ret) { \
SHERPA_ONNX_LOGE("Invalid value '%s' for '%s'", value.c_str(), src_key); \
exit(-1); \
} \
} while (0)
// read a vector of strings
#define SHERPA_ONNX_READ_META_DATA_VEC_STRING(dst, src_key) \
do { \
auto value = \
meta_data.LookupCustomMetadataMapAllocated(src_key, allocator); \
if (!value) { \
SHERPA_ONNX_LOGE("%s does not exist in the metadata", src_key); \
exit(-1); \
} \
SplitStringToVector(value.get(), ",", false, &dst); \
\
if (dst.empty()) { \
SHERPA_ONNX_LOGE("Invalid value %s for %s. Empty vector!", value.get(), \
src_key); \
exit(-1); \
} \
#define SHERPA_ONNX_READ_META_DATA_VEC_STRING(dst, src_key) \
do { \
auto value = LookupCustomModelMetaData(meta_data, src_key, allocator); \
if (value.empty()) { \
SHERPA_ONNX_LOGE("'%s' does not exist in the metadata", src_key); \
exit(-1); \
} \
SplitStringToVector(value.c_str(), ",", false, &dst); \
\
if (dst.empty()) { \
SHERPA_ONNX_LOGE("Invalid value '%s' for '%s'. Empty vector!", \
value.c_str(), src_key); \
exit(-1); \
} \
} while (0)
// read a vector of strings separated by sep
#define SHERPA_ONNX_READ_META_DATA_VEC_STRING_SEP(dst, src_key, sep) \
do { \
auto value = \
meta_data.LookupCustomMetadataMapAllocated(src_key, allocator); \
if (!value) { \
SHERPA_ONNX_LOGE("%s does not exist in the metadata", src_key); \
exit(-1); \
} \
SplitStringToVector(value.get(), sep, false, &dst); \
\
if (dst.empty()) { \
SHERPA_ONNX_LOGE("Invalid value %s for %s. Empty vector!", value.get(), \
src_key); \
exit(-1); \
} \
#define SHERPA_ONNX_READ_META_DATA_VEC_STRING_SEP(dst, src_key, sep) \
do { \
auto value = LookupCustomModelMetaData(meta_data, src_key, allocator); \
if (value.empty()) { \
SHERPA_ONNX_LOGE("'%s' does not exist in the metadata", src_key); \
exit(-1); \
} \
SplitStringToVector(value.c_str(), sep, false, &dst); \
\
if (dst.empty()) { \
SHERPA_ONNX_LOGE("Invalid value '%s' for '%s'. Empty vector!", \
value.c_str(), src_key); \
exit(-1); \
} \
} while (0)
// Read a string
#define SHERPA_ONNX_READ_META_DATA_STR(dst, src_key) \
do { \
auto value = \
meta_data.LookupCustomMetadataMapAllocated(src_key, allocator); \
if (!value) { \
SHERPA_ONNX_LOGE("%s does not exist in the metadata", src_key); \
exit(-1); \
} \
\
dst = value.get(); \
if (dst.empty()) { \
SHERPA_ONNX_LOGE("Invalid value for %s\n", src_key); \
exit(-1); \
} \
#define SHERPA_ONNX_READ_META_DATA_STR(dst, src_key) \
do { \
auto value = LookupCustomModelMetaData(meta_data, src_key, allocator); \
if (value.empty()) { \
SHERPA_ONNX_LOGE("'%s' does not exist in the metadata", src_key); \
exit(-1); \
} \
\
dst = std::move(value); \
if (dst.empty()) { \
SHERPA_ONNX_LOGE("Invalid value for '%s'\n", src_key); \
exit(-1); \
} \
} while (0)
#define SHERPA_ONNX_READ_META_DATA_STR_WITH_DEFAULT(dst, src_key, \
default_value) \
do { \
auto value = \
meta_data.LookupCustomMetadataMapAllocated(src_key, allocator); \
if (!value) { \
dst = default_value; \
} else { \
dst = value.get(); \
if (dst.empty()) { \
SHERPA_ONNX_LOGE("Invalid value for %s\n", src_key); \
exit(-1); \
} \
} \
#define SHERPA_ONNX_READ_META_DATA_STR_ALLOW_EMPTY(dst, src_key) \
do { \
auto value = LookupCustomModelMetaData(meta_data, src_key, allocator); \
\
dst = std::move(value); \
} while (0)
#define SHERPA_ONNX_READ_META_DATA_STR_WITH_DEFAULT(dst, src_key, \
default_value) \
do { \
auto value = LookupCustomModelMetaData(meta_data, src_key, allocator); \
if (value.empty()) { \
dst = default_value; \
} else { \
dst = std::move(value); \
if (dst.empty()) { \
SHERPA_ONNX_LOGE("Invalid value for '%s'\n", src_key); \
exit(-1); \
} \
} \
} while (0)
#define SHERPA_ONNX_EXIT(code) exit(code)
#endif // SHERPA_ONNX_CSRC_MACROS_H_
projects/llm_framework/include/sherpa/sherpa-onnx/csrc/melo-tts-lexicon.h
+13
View File
@@ -22,6 +22,19 @@ class MeloTtsLexicon : public OfflineTtsFrontend {
const std::string &dict_dir,
const OfflineTtsVitsModelMetaData &meta_data, bool debug);
MeloTtsLexicon(const std::string &lexicon, const std::string &tokens,
const OfflineTtsVitsModelMetaData &meta_data, bool debug);
template <typename Manager>
MeloTtsLexicon(Manager *mgr, const std::string &lexicon,
const std::string &tokens, const std::string &dict_dir,
const OfflineTtsVitsModelMetaData &meta_data, bool debug);
template <typename Manager>
MeloTtsLexicon(Manager *mgr, const std::string &lexicon,
const std::string &tokens,
const OfflineTtsVitsModelMetaData &meta_data, bool debug);
std::vector<TokenIDs> ConvertTextToTokenIds(
const std::string &text,
const std::string &unused_voice = "") const override;
projects/llm_framework/include/sherpa/sherpa-onnx/csrc/offline-ctc-model.h
+6 -8
View File
@@ -8,11 +8,6 @@
#include <string>
#include <vector>
#if __ANDROID_API__ >= 9
#include "android/asset_manager.h"
#include "android/asset_manager_jni.h"
#endif
#include "onnxruntime_cxx_api.h" // NOLINT
#include "sherpa-onnx/csrc/offline-model-config.h"
@@ -25,10 +20,9 @@ class OfflineCtcModel {
static std::unique_ptr<OfflineCtcModel> Create(
const OfflineModelConfig &config);
#if __ANDROID_API__ >= 9
template <typename Manager>
static std::unique_ptr<OfflineCtcModel> Create(
AAssetManager *mgr, const OfflineModelConfig &config);
#endif
Manager *mgr, const OfflineModelConfig &config);
/** Run the forward method of the model.
*
@@ -66,6 +60,10 @@ class OfflineCtcModel {
// Return true if the model supports batch size > 1
virtual bool SupportBatchProcessing() const { return true; }
// return true for models from https://github.com/salute-developers/GigaAM
// return false otherwise
virtual bool IsGigaAM() const { return false; }
};
} // namespace sherpa_onnx
projects/llm_framework/include/sherpa/sherpa-onnx/csrc/offline-lm.h
+2 -8
View File
@@ -8,11 +8,6 @@
#include <memory>
#include <vector>
#if __ANDROID_API__ >= 9
#include "android/asset_manager.h"
#include "android/asset_manager_jni.h"
#endif
#include "onnxruntime_cxx_api.h" // NOLINT
#include "sherpa-onnx/csrc/hypothesis.h"
#include "sherpa-onnx/csrc/offline-lm-config.h"
@@ -25,10 +20,9 @@ class OfflineLM {
static std::unique_ptr<OfflineLM> Create(const OfflineLMConfig &config);
#if __ANDROID_API__ >= 9
static std::unique_ptr<OfflineLM> Create(AAssetManager *mgr,
template <typename Manager>
static std::unique_ptr<OfflineLM> Create(Manager *mgr,
const OfflineLMConfig &config);
#endif
/** Rescore a batch of sentences.
*
projects/llm_framework/include/sherpa/sherpa-onnx/csrc/offline-model-config.h
+4
View File
@@ -6,6 +6,7 @@
#include <string>
#include "sherpa-onnx/csrc/offline-moonshine-model-config.h"
#include "sherpa-onnx/csrc/offline-nemo-enc-dec-ctc-model-config.h"
#include "sherpa-onnx/csrc/offline-paraformer-model-config.h"
#include "sherpa-onnx/csrc/offline-sense-voice-model-config.h"
@@ -26,6 +27,7 @@ struct OfflineModelConfig {
OfflineZipformerCtcModelConfig zipformer_ctc;
OfflineWenetCtcModelConfig wenet_ctc;
OfflineSenseVoiceModelConfig sense_voice;
OfflineMoonshineModelConfig moonshine;
std::string telespeech_ctc;
std::string tokens;
@@ -56,6 +58,7 @@ struct OfflineModelConfig {
const OfflineZipformerCtcModelConfig &zipformer_ctc,
const OfflineWenetCtcModelConfig &wenet_ctc,
const OfflineSenseVoiceModelConfig &sense_voice,
const OfflineMoonshineModelConfig &moonshine,
const std::string &telespeech_ctc,
const std::string &tokens, int32_t num_threads, bool debug,
const std::string &provider, const std::string &model_type,
@@ -69,6 +72,7 @@ struct OfflineModelConfig {
zipformer_ctc(zipformer_ctc),
wenet_ctc(wenet_ctc),
sense_voice(sense_voice),
moonshine(moonshine),
telespeech_ctc(telespeech_ctc),
tokens(tokens),
num_threads(num_threads),
projects/llm_framework/include/sherpa/sherpa-onnx/csrc/offline-moonshine-decoder.h
+34
View File
@@ -0,0 +1,34 @@
// sherpa-onnx/csrc/offline-moonshine-decoder.h
//
// Copyright (c) 2023 Xiaomi Corporation
#ifndef SHERPA_ONNX_CSRC_OFFLINE_MOONSHINE_DECODER_H_
#define SHERPA_ONNX_CSRC_OFFLINE_MOONSHINE_DECODER_H_
#include <vector>
#include "onnxruntime_cxx_api.h" // NOLINT
namespace sherpa_onnx {
struct OfflineMoonshineDecoderResult {
/// The decoded token IDs
std::vector<int32_t> tokens;
};
class OfflineMoonshineDecoder {
public:
virtual ~OfflineMoonshineDecoder() = default;
/** Run beam search given the output from the moonshine encoder model.
*
* @param encoder_out A 3-D tensor of shape (batch_size, T, dim)
* @return Return a vector of size `N` containing the decoded results.
*/
virtual std::vector<OfflineMoonshineDecoderResult> Decode(
Ort::Value encoder_out) = 0;
};
} // namespace sherpa_onnx
#endif // SHERPA_ONNX_CSRC_OFFLINE_MOONSHINE_DECODER_H_
projects/llm_framework/include/sherpa/sherpa-onnx/csrc/offline-moonshine-greedy-search-decoder.h
+29
View File
@@ -0,0 +1,29 @@
// sherpa-onnx/csrc/offline-moonshine-greedy-search-decoder.h
//
// Copyright (c) 2024 Xiaomi Corporation
#ifndef SHERPA_ONNX_CSRC_OFFLINE_MOONSHINE_GREEDY_SEARCH_DECODER_H_
#define SHERPA_ONNX_CSRC_OFFLINE_MOONSHINE_GREEDY_SEARCH_DECODER_H_
#include <vector>
#include "sherpa-onnx/csrc/offline-moonshine-decoder.h"
#include "sherpa-onnx/csrc/offline-moonshine-model.h"
namespace sherpa_onnx {
class OfflineMoonshineGreedySearchDecoder : public OfflineMoonshineDecoder {
public:
explicit OfflineMoonshineGreedySearchDecoder(OfflineMoonshineModel *model)
: model_(model) {}
std::vector<OfflineMoonshineDecoderResult> Decode(
Ort::Value encoder_out) override;
private:
OfflineMoonshineModel *model_; // not owned
};
} // namespace sherpa_onnx
#endif // SHERPA_ONNX_CSRC_OFFLINE_MOONSHINE_GREEDY_SEARCH_DECODER_H_
projects/llm_framework/include/sherpa/sherpa-onnx/csrc/offline-moonshine-model-config.h
+37
View File
@@ -0,0 +1,37 @@
// sherpa-onnx/csrc/offline-moonshine-model-config.h
//
// Copyright (c) 2024 Xiaomi Corporation
#ifndef SHERPA_ONNX_CSRC_OFFLINE_MOONSHINE_MODEL_CONFIG_H_
#define SHERPA_ONNX_CSRC_OFFLINE_MOONSHINE_MODEL_CONFIG_H_
#include <string>
#include "sherpa-onnx/csrc/parse-options.h"
namespace sherpa_onnx {
struct OfflineMoonshineModelConfig {
std::string preprocessor;
std::string encoder;
std::string uncached_decoder;
std::string cached_decoder;
OfflineMoonshineModelConfig() = default;
OfflineMoonshineModelConfig(const std::string &preprocessor,
const std::string &encoder,
const std::string &uncached_decoder,
const std::string &cached_decoder)
: preprocessor(preprocessor),
encoder(encoder),
uncached_decoder(uncached_decoder),
cached_decoder(cached_decoder) {}
void Register(ParseOptions *po);
bool Validate() const;
std::string ToString() const;
};
} // namespace sherpa_onnx
#endif // SHERPA_ONNX_CSRC_OFFLINE_MOONSHINE_MODEL_CONFIG_H_
projects/llm_framework/include/sherpa/sherpa-onnx/csrc/offline-moonshine-model.h
+87
View File
@@ -0,0 +1,87 @@
// sherpa-onnx/csrc/offline-moonshine-model.h
//
// Copyright (c) 2024 Xiaomi Corporation
#ifndef SHERPA_ONNX_CSRC_OFFLINE_MOONSHINE_MODEL_H_
#define SHERPA_ONNX_CSRC_OFFLINE_MOONSHINE_MODEL_H_
#include <memory>
#include <string>
#include <utility>
#include <vector>
#include "onnxruntime_cxx_api.h" // NOLINT
#include "sherpa-onnx/csrc/offline-model-config.h"
namespace sherpa_onnx {
// please see
// https://github.com/k2-fsa/sherpa-onnx/blob/master/scripts/moonshine/test.py
class OfflineMoonshineModel {
public:
explicit OfflineMoonshineModel(const OfflineModelConfig &config);
template <typename Manager>
OfflineMoonshineModel(Manager *mgr, const OfflineModelConfig &config);
~OfflineMoonshineModel();
/** Run the preprocessor model.
*
* @param audio A float32 tensor of shape (batch_size, num_samples)
*
* @return Return a float32 tensor of shape (batch_size, T, dim) that
* can be used as the input of ForwardEncoder()
*/
Ort::Value ForwardPreprocessor(Ort::Value audio) const;
/** Run the encoder model.
*
* @param features A float32 tensor of shape (batch_size, T, dim)
* @param features_len A int32 tensor of shape (batch_size,)
* @returns A float32 tensor of shape (batch_size, T, dim).
*/
Ort::Value ForwardEncoder(Ort::Value features, Ort::Value features_len) const;
/** Run the uncached decoder.
*
* @param token A int32 tensor of shape (batch_size, num_tokens)
* @param seq_len A int32 tensor of shape (batch_size,) containing number
* of predicted tokens so far
* @param encoder_out A float32 tensor of shape (batch_size, T, dim)
*
* @returns Return a pair:
*
* - logits, a float32 tensor of shape (batch_size, 1, dim)
* - states, a list of states
*/
std::pair<Ort::Value, std::vector<Ort::Value>> ForwardUnCachedDecoder(
Ort::Value token, Ort::Value seq_len, Ort::Value encoder_out) const;
/** Run the cached decoder.
*
* @param token A int32 tensor of shape (batch_size, num_tokens)
* @param seq_len A int32 tensor of shape (batch_size,) containing number
* of predicted tokens so far
* @param encoder_out A float32 tensor of shape (batch_size, T, dim)
* @param states A list of previous states
*
* @returns Return a pair:
* - logits, a float32 tensor of shape (batch_size, 1, dim)
* - states, a list of new states
*/
std::pair<Ort::Value, std::vector<Ort::Value>> ForwardCachedDecoder(
Ort::Value token, Ort::Value seq_len, Ort::Value encoder_out,
std::vector<Ort::Value> states) const;
/** Return an allocator for allocating memory
*/
OrtAllocator *Allocator() const;
private:
class Impl;
std::unique_ptr<Impl> impl_;
};
} // namespace sherpa_onnx
#endif // SHERPA_ONNX_CSRC_OFFLINE_MOONSHINE_MODEL_H_

Some files were not shown because too many files have changed in this diff Show More