ngram_model.h
Go to the documentation of this file.
/* -*- c-basic-offset: 4; indent-tabs-mode: nil -*- */
/* ====================================================================
* Copyright (c) 2007 Carnegie Mellon University. All rights
* reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
*
* This work was supported in part by funding from the Defense Advanced
* Research Projects Agency and the National Science Foundation of the
* United States of America, and the CMU Sphinx Speech Consortium.
*
* THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND
* ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
* THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY
* NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
* ====================================================================
*
*/
#ifndef __NGRAM_MODEL_H__
#define __NGRAM_MODEL_H__
#include <stdarg.h>
/* Win32/WinCE DLL gunk */
#include <sphinxbase/sphinxbase_export.h>
#include <sphinxbase/prim_type.h>
#include <sphinxbase/cmd_ln.h>
#include <sphinxbase/logmath.h>
#include <sphinxbase/mmio.h>
#ifdef __cplusplus
extern "C" {
#endif
#if 0
/* Fool Emacs. */
}
#endif
typedef struct ngram_model_s ngram_model_t;
typedef struct ngram_class_s ngram_class_t;
typedef enum ngram_file_type_e {
NGRAM_INVALID = -1,
NGRAM_AUTO,
NGRAM_ARPA,
NGRAM_DMP,
NGRAM_DMP32,
} ngram_file_type_t;
#define NGRAM_INVALID_WID -1
SPHINXBASE_EXPORT
ngram_model_t *ngram_model_read(cmd_ln_t *config,
const char *file_name,
ngram_file_type_t file_type,
logmath_t *lmath);
SPHINXBASE_EXPORT
int ngram_model_write(ngram_model_t *model, const char *file_name,
ngram_file_type_t format);
SPHINXBASE_EXPORT
ngram_file_type_t ngram_file_name_to_type(const char *file_name);
SPHINXBASE_EXPORT
ngram_file_type_t ngram_str_to_type(const char *str_name);
SPHINXBASE_EXPORT
char const *ngram_type_to_str(int type);
SPHINXBASE_EXPORT
ngram_model_t *ngram_model_retain(ngram_model_t *model);
SPHINXBASE_EXPORT
int ngram_model_free(ngram_model_t *model);
SPHINXBASE_EXPORT
int ngram_model_recode(ngram_model_t *model, const char *from, const char *to);
typedef enum ngram_case_e {
NGRAM_UPPER,
NGRAM_LOWER
} ngram_case_t;
SPHINXBASE_EXPORT
int ngram_model_casefold(ngram_model_t *model, int kase);
SPHINXBASE_EXPORT
int ngram_model_apply_weights(ngram_model_t *model,
float32 lw, float32 wip, float32 uw);
SPHINXBASE_EXPORT
float32 ngram_model_get_weights(ngram_model_t *model, int32 *out_log_wip,
int32 *out_log_uw);
SPHINXBASE_EXPORT
int32 ngram_score(ngram_model_t *model, const char *word, ...);
SPHINXBASE_EXPORT
int32 ngram_tg_score(ngram_model_t *model,
int32 w3, int32 w2, int32 w1,
int32 *n_used);
SPHINXBASE_EXPORT
int32 ngram_bg_score(ngram_model_t *model,
int32 w2, int32 w1,
int32 *n_used);
SPHINXBASE_EXPORT
int32 ngram_ng_score(ngram_model_t *model, int32 wid, int32 *history,
int32 n_hist, int32 *n_used);
SPHINXBASE_EXPORT
int32 ngram_prob(ngram_model_t *model, const char *word, ...);
SPHINXBASE_EXPORT
int32 ngram_ng_prob(ngram_model_t *model, int32 wid, int32 *history,
int32 n_hist, int32 *n_used);
SPHINXBASE_EXPORT
int32 ngram_score_to_prob(ngram_model_t *model, int32 score);
SPHINXBASE_EXPORT
int32 ngram_wid(ngram_model_t *model, const char *word);
SPHINXBASE_EXPORT
const char *ngram_word(ngram_model_t *model, int32 wid);
SPHINXBASE_EXPORT
int32 ngram_unknown_wid(ngram_model_t *model);
SPHINXBASE_EXPORT
int32 ngram_zero(ngram_model_t *model);
SPHINXBASE_EXPORT
int32 ngram_model_get_size(ngram_model_t *model);
SPHINXBASE_EXPORT
int32 const *ngram_model_get_counts(ngram_model_t *model);
typedef struct ngram_iter_s ngram_iter_t;
SPHINXBASE_EXPORT
ngram_iter_t *ngram_model_mgrams(ngram_model_t *model, int m);
SPHINXBASE_EXPORT
ngram_iter_t *ngram_iter(ngram_model_t *model, const char *word, ...);
SPHINXBASE_EXPORT
ngram_iter_t *ngram_ng_iter(ngram_model_t *model, int32 wid, int32 *history, int32 n_hist);
SPHINXBASE_EXPORT
int32 const *ngram_iter_get(ngram_iter_t *itor,
int32 *out_score,
int32 *out_bowt);
SPHINXBASE_EXPORT
ngram_iter_t *ngram_iter_successors(ngram_iter_t *itor);
SPHINXBASE_EXPORT
ngram_iter_t *ngram_iter_next(ngram_iter_t *itor);
SPHINXBASE_EXPORT
void ngram_iter_free(ngram_iter_t *itor);
SPHINXBASE_EXPORT
int32 ngram_model_add_word(ngram_model_t *model,
const char *word, float32 weight);
SPHINXBASE_EXPORT
int32 ngram_model_read_classdef(ngram_model_t *model,
const char *file_name);
SPHINXBASE_EXPORT
int32 ngram_model_add_class(ngram_model_t *model,
const char *classname,
float32 classweight,
char **words,
const float32 *weights,
int32 n_words);
SPHINXBASE_EXPORT
int32 ngram_model_add_class_word(ngram_model_t *model,
const char *classname,
const char *word,
float32 weight);
SPHINXBASE_EXPORT
ngram_model_t *ngram_model_set_init(cmd_ln_t *config,
ngram_model_t **models,
char **names,
const float32 *weights,
int32 n_models);
SPHINXBASE_EXPORT
ngram_model_t *ngram_model_set_read(cmd_ln_t *config,
const char *lmctlfile,
logmath_t *lmath);
SPHINXBASE_EXPORT
int32 ngram_model_set_count(ngram_model_t *set);
typedef struct ngram_model_set_iter_s ngram_model_set_iter_t;
SPHINXBASE_EXPORT
ngram_model_set_iter_t *ngram_model_set_iter(ngram_model_t *set);
SPHINXBASE_EXPORT
ngram_model_set_iter_t *ngram_model_set_iter_next(ngram_model_set_iter_t *itor);
SPHINXBASE_EXPORT
void ngram_model_set_iter_free(ngram_model_set_iter_t *itor);
SPHINXBASE_EXPORT
ngram_model_t *ngram_model_set_iter_model(ngram_model_set_iter_t *itor,
char const **lmname);
SPHINXBASE_EXPORT
ngram_model_t *ngram_model_set_select(ngram_model_t *set,
const char *name);
SPHINXBASE_EXPORT
ngram_model_t *ngram_model_set_lookup(ngram_model_t *set,
const char *name);
SPHINXBASE_EXPORT
const char *ngram_model_set_current(ngram_model_t *set);
SPHINXBASE_EXPORT
ngram_model_t *ngram_model_set_interp(ngram_model_t *set,
const char **names,
const float32 *weights);
SPHINXBASE_EXPORT
ngram_model_t *ngram_model_set_add(ngram_model_t *set,
ngram_model_t *model,
const char *name,
float32 weight,
int reuse_widmap);
SPHINXBASE_EXPORT
ngram_model_t *ngram_model_set_remove(ngram_model_t *set,
const char *name,
int reuse_widmap);
SPHINXBASE_EXPORT
void ngram_model_set_map_words(ngram_model_t *set,
const char **words,
int32 n_words);
SPHINXBASE_EXPORT
int32 ngram_model_set_current_wid(ngram_model_t *set,
int32 set_wid);
SPHINXBASE_EXPORT
int32 ngram_model_set_known_wid(ngram_model_t *set, int32 set_wid);
SPHINXBASE_EXPORT
void ngram_model_flush(ngram_model_t *lm);
#ifdef __cplusplus
}
#endif
#endif /* __NGRAM_MODEL_H__ */
ngram_ng_prob
SPHINXBASE_EXPORT int32 ngram_ng_prob(ngram_model_t *model, int32 wid, int32 *history, int32 n_hist, int32 *n_used)
Quick "raw" probability lookup for a general N-Gram.
Definition: ngram_model.c:570
ngram_model_set_init
SPHINXBASE_EXPORT ngram_model_t * ngram_model_set_init(cmd_ln_t *config, ngram_model_t **models, char **names, const float32 *weights, int32 n_models)
Create a set of language models sharing a common space of word IDs.
Definition: ngram_model_set.c:121
cmd_ln.h
Command-line and other configurationparsing and handling.
ngram_model_read
SPHINXBASE_EXPORT ngram_model_t * ngram_model_read(cmd_ln_t *config, const char *file_name, ngram_file_type_t file_type, logmath_t *lmath)
Read an N-Gram model from a file on disk.
Definition: ngram_model.c:121
ngram_model_write
SPHINXBASE_EXPORT int ngram_model_write(ngram_model_t *model, const char *file_name, ngram_file_type_t format)
Write an N-Gram model to disk.
Definition: ngram_model.c:167
ngram_model_casefold
SPHINXBASE_EXPORT int ngram_model_casefold(ngram_model_t *model, int kase)
Case-fold word strings in an N-Gram model.
Definition: ngram_model.c:299
ngram_model_set_lookup
SPHINXBASE_EXPORT ngram_model_t * ngram_model_set_lookup(ngram_model_t *set, const char *name)
Look up a language model by name from a set.
Definition: ngram_model_set.c:405
ngram_unknown_wid
SPHINXBASE_EXPORT int32 ngram_unknown_wid(ngram_model_t *model)
Get the unknown word ID for a language model.
Definition: ngram_model.c:644
ngram_iter_free
SPHINXBASE_EXPORT void ngram_iter_free(ngram_iter_t *itor)
Terminate an M-gram iterator.
Definition: ngram_model.c:764
ngram_model_get_counts
SPHINXBASE_EXPORT int32 const * ngram_model_get_counts(ngram_model_t *model)
Get the counts of the various N-grams in the model.
Definition: ngram_model.c:671
ngram_model_set_iter_next
SPHINXBASE_EXPORT ngram_model_set_iter_t * ngram_model_set_iter_next(ngram_model_set_iter_t *itor)
Move to the next language model in a set.
Definition: ngram_model_set.c:381
ngram_wid
SPHINXBASE_EXPORT int32 ngram_wid(ngram_model_t *model, const char *word)
Look up numerical word ID.
Definition: ngram_model.c:771
ngram_iter_get
SPHINXBASE_EXPORT int32 const * ngram_iter_get(ngram_iter_t *itor, int32 *out_score, int32 *out_bowt)
Get information from the current M-gram in an iterator.
Definition: ngram_model.c:750
logmath_s
Definition: logmath.c:49
ngram_iter_next
SPHINXBASE_EXPORT ngram_iter_t * ngram_iter_next(ngram_iter_t *itor)
Advance an M-gram iterator.
Definition: ngram_model.c:758
ngram_model_set_read
SPHINXBASE_EXPORT ngram_model_t * ngram_model_set_read(cmd_ln_t *config, const char *lmctlfile, logmath_t *lmath)
Read a set of language models from a control file.
Definition: ngram_model_set.c:182
ngram_iter_s::m
int16 m
Order of history.
Definition: ngram_model_internal.h:196
ngram_tg_score
SPHINXBASE_EXPORT int32 ngram_tg_score(ngram_model_t *model, int32 w3, int32 w2, int32 w1, int32 *n_used)
Quick trigram score lookup.
Definition: ngram_model.c:555
ngram_model_free
SPHINXBASE_EXPORT int ngram_model_free(ngram_model_t *model)
Release memory associated with an N-Gram model.
Definition: ngram_model.c:254
prim_type.h
Basic type definitions used in Sphinx.
ngram_iter_successors
SPHINXBASE_EXPORT ngram_iter_t * ngram_iter_successors(ngram_iter_t *itor)
Iterate over all M-gram successors of an M-1-gram.
Definition: ngram_model.c:741
ngram_model_set_count
SPHINXBASE_EXPORT int32 ngram_model_set_count(ngram_model_t *set)
Returns the number of language models in a set.
Definition: ngram_model_set.c:361
NGRAM_DMP
Sphinx .DMP format.
Definition: ngram_model.h:80
ngram_model_set_add
SPHINXBASE_EXPORT ngram_model_t * ngram_model_set_add(ngram_model_t *set, ngram_model_t *model, const char *name, float32 weight, int reuse_widmap)
Add a language model to a set.
Definition: ngram_model_set.c:520
NGRAM_ARPA
ARPABO text format (the standard).
Definition: ngram_model.h:79
ngram_prob
SPHINXBASE_EXPORT int32 ngram_prob(ngram_model_t *model, const char *word,...)
Get the "raw" log-probability for a general N-Gram.
Definition: ngram_model.c:600
ngram_model_set_iter_s
Iterator over a model set.
Definition: ngram_model_set.h:66
ngram_model_set_remove
SPHINXBASE_EXPORT ngram_model_t * ngram_model_set_remove(ngram_model_t *set, const char *name, int reuse_widmap)
Remove a language model from a set.
Definition: ngram_model_set.c:580
ngram_model_apply_weights
SPHINXBASE_EXPORT int ngram_model_apply_weights(ngram_model_t *model, float32 lw, float32 wip, float32 uw)
Apply a language weight, insertion penalty, and unigram weight to a language model.
Definition: ngram_model.c:477
ngram_model_retain
SPHINXBASE_EXPORT ngram_model_t * ngram_model_retain(ngram_model_t *model)
Retain ownership of an N-Gram model.
Definition: ngram_model.c:239
ngram_score
SPHINXBASE_EXPORT int32 ngram_score(ngram_model_t *model, const char *word,...)
Get the score (scaled, interpolated log-probability) for a general N-Gram.
Definition: ngram_model.c:524
ngram_model_get_size
SPHINXBASE_EXPORT int32 ngram_model_get_size(ngram_model_t *model)
Get the order of the N-gram model (i.e.
Definition: ngram_model.c:663
ngram_type_to_str
SPHINXBASE_EXPORT char const * ngram_type_to_str(int type)
Get the canonical name for an N-Gram file type.
Definition: ngram_model.c:107
ngram_model_set_map_words
SPHINXBASE_EXPORT void ngram_model_set_map_words(ngram_model_t *set, const char **words, int32 n_words)
Set the word-to-ID mapping for this model set.
Definition: ngram_model_set.c:637
ngram_bg_score
SPHINXBASE_EXPORT int32 ngram_bg_score(ngram_model_t *model, int32 w2, int32 w1, int32 *n_used)
Quick bigram score lookup.
Definition: ngram_model.c:564
ngram_model_add_word
SPHINXBASE_EXPORT int32 ngram_model_add_word(ngram_model_t *model, const char *word, float32 weight)
Add a word (unigram) to the language model.
Definition: ngram_model.c:832
ngram_model_set_iter_model
SPHINXBASE_EXPORT ngram_model_t * ngram_model_set_iter_model(ngram_model_set_iter_t *itor, char const **lmname)
Get language model and associated name from an iterator.
Definition: ngram_model_set.c:397
ngram_model_get_weights
SPHINXBASE_EXPORT float32 ngram_model_get_weights(ngram_model_t *model, int32 *out_log_wip, int32 *out_log_uw)
Get the current weights from a language model.
Definition: ngram_model.c:484
ngram_model_add_class
SPHINXBASE_EXPORT int32 ngram_model_add_class(ngram_model_t *model, const char *classname, float32 classweight, char **words, const float32 *weights, int32 n_words)
Add a new class to a language model.
Definition: ngram_model.c:1005
ngram_str_to_type
SPHINXBASE_EXPORT ngram_file_type_t ngram_str_to_type(const char *str_name)
Get the N-Gram file type from a string.
Definition: ngram_model.c:97
ngram_ng_iter
SPHINXBASE_EXPORT ngram_iter_t * ngram_ng_iter(ngram_model_t *model, int32 wid, int32 *history, int32 n_hist)
Get an iterator over M-grams pointing to the specified M-gram.
Definition: ngram_model.c:731
ngram_model_read_classdef
SPHINXBASE_EXPORT int32 ngram_model_read_classdef(ngram_model_t *model, const char *file_name)
Read a class definition file and add classes to a language model.
Definition: ngram_model.c:1199
ngram_model_set_interp
SPHINXBASE_EXPORT ngram_model_t * ngram_model_set_interp(ngram_model_t *set, const char **names, const float32 *weights)
Set interpolation weights for a set and enables interpolation.
Definition: ngram_model_set.c:489
ngram_model_set_iter
SPHINXBASE_EXPORT ngram_model_set_iter_t * ngram_model_set_iter(ngram_model_t *set)
Begin iterating over language models in a set.
Definition: ngram_model_set.c:368
ngram_model_flush
SPHINXBASE_EXPORT void ngram_model_flush(ngram_model_t *lm)
Flush any cached N-Gram information.
Definition: ngram_model.c:247
cmd_ln_t
Opaque structure used to hold the results of command-line parsing.
ngram_file_type_t
enum ngram_file_type_e ngram_file_type_t
File types for N-Gram files.
ngram_iter
SPHINXBASE_EXPORT ngram_iter_t * ngram_iter(ngram_model_t *model, const char *word,...)
Get an iterator over M-grams pointing to the specified M-gram.
Definition: ngram_model.c:702
ngram_file_type_e
ngram_file_type_e
File types for N-Gram files.
Definition: ngram_model.h:76
ngram_model_set_iter_free
SPHINXBASE_EXPORT void ngram_model_set_iter_free(ngram_model_set_iter_t *itor)
Finish iteration over a langauge model set.
Definition: ngram_model_set.c:391
NGRAM_DMP32
Sphinx .DMP32 format (NOT SUPPORTED)
Definition: ngram_model.h:81
ngram_file_name_to_type
SPHINXBASE_EXPORT ngram_file_type_t ngram_file_name_to_type(const char *file_name)
Guess the file type for an N-Gram model from the filename.
Definition: ngram_model.c:64
ngram_model_set_current
SPHINXBASE_EXPORT const char * ngram_model_set_current(ngram_model_t *set)
Get the current language model name, if any.
Definition: ngram_model_set.c:445
ngram_iter_s
Base iterator structure for N-grams.
Definition: ngram_model_internal.h:193
NGRAM_INVALID
Not a valid file type.
Definition: ngram_model.h:77
ngram_class_s
Implementation of ngram_class_t.
Definition: ngram_model_internal.h:83
ngram_case_e
ngram_case_e
Constants for case folding.
Definition: ngram_model.h:184
ngram_model_s
Common implementation of ngram_model_t.
Definition: ngram_model_internal.h:55
ngram_score_to_prob
SPHINXBASE_EXPORT int32 ngram_score_to_prob(ngram_model_t *model, int32 score)
Convert score to "raw" log-probability.
Definition: ngram_model.c:631
ngram_word
SPHINXBASE_EXPORT const char * ngram_word(ngram_model_t *model, int32 wid)
Look up word string for numerical word ID.
Definition: ngram_model.c:782
ngram_zero
SPHINXBASE_EXPORT int32 ngram_zero(ngram_model_t *model)
Get the "zero" log-probability value for a language model.
Definition: ngram_model.c:657
mmio.h
Memory-mapped I/O wrappers for files.
ngram_model_set_select
SPHINXBASE_EXPORT ngram_model_t * ngram_model_set_select(ngram_model_t *set, const char *name)
Select a single language model from a set for scoring.
Definition: ngram_model_set.c:428
logmath.h
Fast integer logarithmic addition operations.
ngram_model_set_current_wid
SPHINXBASE_EXPORT int32 ngram_model_set_current_wid(ngram_model_t *set, int32 set_wid)
Query the word-ID mapping for the current language model.
Definition: ngram_model_set.c:456
ngram_case_t
enum ngram_case_e ngram_case_t
Constants for case folding.
ngram_model_add_class_word
SPHINXBASE_EXPORT int32 ngram_model_add_class_word(ngram_model_t *model, const char *classname, const char *word, float32 weight)
Add a word to a class in a language model.
Definition: ngram_model.c:955
ngram_model_mgrams
SPHINXBASE_EXPORT ngram_iter_t * ngram_model_mgrams(ngram_model_t *model, int m)
Iterate over all M-grams.
Definition: ngram_model.c:689
ngram_model_set_known_wid
SPHINXBASE_EXPORT int32 ngram_model_set_known_wid(ngram_model_t *set, int32 set_wid)
Test whether a word ID corresponds to a known word in the current state of the language model set...
Definition: ngram_model_set.c:468
NGRAM_AUTO
Determine file type automatically.
Definition: ngram_model.h:78
ngram_ng_score
SPHINXBASE_EXPORT int32 ngram_ng_score(ngram_model_t *model, int32 wid, int32 *history, int32 n_hist, int32 *n_used)
Quick general N-Gram score lookup.
Definition: ngram_model.c:494
ngram_model_recode
SPHINXBASE_EXPORT int ngram_model_recode(ngram_model_t *model, const char *from, const char *to)
Re-encode word strings in an N-Gram model.
Definition: ngram_model.c:470