ngram_model_internal.h
/* -*- c-basic-offset: 4; indent-tabs-mode: nil -*- */
/* ====================================================================
* Copyright (c) 1999-2007 Carnegie Mellon University. All rights
* reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
*
* This work was supported in part by funding from the Defense Advanced
* Research Projects Agency and the National Science Foundation of the
* United States of America, and the CMU Sphinx Speech Consortium.
*
* THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND
* ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
* THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY
* NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
* ====================================================================
*
*/
/*
* \file ngram_model_internal.h Internal structures for N-Gram models
*
* Author: David Huggins-Daines <dhuggins@cs.cmu.edu>
*/
#ifndef __NGRAM_MODEL_INTERNAL_H__
#define __NGRAM_MODEL_INTERNAL_H__
#include "sphinxbase/ngram_model.h"
#include "sphinxbase/hash_table.h"
struct ngram_model_s {
int refcount;
int32 *n_counts;
int32 n_1g_alloc;
int32 n_words;
uint8 n;
uint8 n_classes;
uint8 writable;
uint8 flags;
logmath_t *lmath;
float32 lw;
int32 log_wip;
int32 log_uw;
int32 log_uniform;
int32 log_uniform_weight;
int32 log_zero;
char **word_str;
hash_table_t *wid;
int32 *tmp_wids;
struct ngram_class_s **classes;
struct ngram_funcs_s *funcs;
};
struct ngram_class_s {
int32 tag_wid;
int32 start_wid;
int32 n_words;
int32 *prob1;
struct ngram_hash_s {
int32 wid;
int32 prob1;
int32 next;
} *nword_hash;
int32 n_hash;
int32 n_hash_inuse;
};
#define NGRAM_HASH_SIZE 128
#define NGRAM_BASEWID(wid) ((wid)&0xffffff)
#define NGRAM_CLASSID(wid) (((wid)>>24) & 0x7f)
#define NGRAM_CLASSWID(wid,classid) (((classid)<<24) | 0x80000000 | (wid))
#define NGRAM_IS_CLASSWID(wid) ((wid)&0x80000000)
#define UG_ALLOC_STEP 10
typedef struct ngram_funcs_s {
void (*free)(ngram_model_t *model);
int (*apply_weights)(ngram_model_t *model,
float32 lw,
float32 wip,
float32 uw);
int32 (*score)(ngram_model_t *model,
int32 wid,
int32 *history,
int32 n_hist,
int32 *n_used);
int32 (*raw_score)(ngram_model_t *model,
int32 wid,
int32 *history,
int32 n_hist,
int32 *n_used);
int32 (*add_ug)(ngram_model_t *model,
int32 wid, int32 lweight);
void (*flush)(ngram_model_t *model);
ngram_iter_t * (*iter)(ngram_model_t *model, int32 wid, int32 *history, int32 n_hist);
ngram_iter_t * (*mgrams)(ngram_model_t *model, int32 m);
ngram_iter_t * (*successors)(ngram_iter_t *itor);
int32 const * (*iter_get)(ngram_iter_t *itor,
int32 *out_score,
int32 *out_bowt);
ngram_iter_t * (*iter_next)(ngram_iter_t *itor);
void (*iter_free)(ngram_iter_t *itor);
} ngram_funcs_t;
struct ngram_iter_s {
ngram_model_t *model;
int32 *wids;
int16 m;
int16 successor;
};
typedef struct classdef_s {
char **words;
float32 *weights;
int32 n_words;
} classdef_t;
int32
ngram_model_init(ngram_model_t *model,
ngram_funcs_t *funcs,
logmath_t *lmath,
int32 n, int32 n_unigram);
ngram_model_t *ngram_model_arpa_read(cmd_ln_t *config,
const char *file_name,
logmath_t *lmath);
ngram_model_t *ngram_model_dmp_read(cmd_ln_t *config,
const char *file_name,
logmath_t *lmath);
ngram_model_t *ngram_model_dmp32_read(cmd_ln_t *config,
const char *file_name,
logmath_t *lmath);
int ngram_model_arpa_write(ngram_model_t *model,
const char *file_name);
int ngram_model_dmp_write(ngram_model_t *model,
const char *file_name);
int32 read_classdef_file(hash_table_t *classes, const char *classdef_file);
void classdef_free(classdef_t *classdef);
ngram_class_t *ngram_class_new(ngram_model_t *model, int32 tag_wid,
int32 start_wid, glist_t classwords);
void ngram_class_free(ngram_class_t *lmclass);
int32 ngram_class_prob(ngram_class_t *lmclass, int32 wid);
void ngram_iter_init(ngram_iter_t *itor, ngram_model_t *model,
int m, int successor);
#endif /* __NGRAM_MODEL_INTERNAL_H__ */
ngram_model_s::funcs
struct ngram_funcs_s * funcs
Implementation-specific methods.
Definition: ngram_model_internal.h:77
ngram_class_s::ngram_hash_s::next
int32 next
Index of next bucket (or -1 for no collision)
Definition: ngram_model_internal.h:94
ngram_model_s::log_uniform
int32 log_uniform
Log of uniform (0-gram) probability.
Definition: ngram_model_internal.h:70
ngram_model_s::tmp_wids
int32 * tmp_wids
Temporary array of word IDs for ngram_model_get_ngram()
Definition: ngram_model_internal.h:75
ngram_model_s::wid
hash_table_t * wid
Mapping of unigram names to word IDs.
Definition: ngram_model_internal.h:74
ngram_model_s::word_str
char ** word_str
Unigram names.
Definition: ngram_model_internal.h:73
ngram_model_s::log_uniform_weight
int32 log_uniform_weight
Log of uniform weight (i.e.
Definition: ngram_model_internal.h:71
logmath_s
Definition: logmath.c:49
ngram_class_s::n_hash
int32 n_hash
Number of buckets in nword_hash (power of 2)
Definition: ngram_model_internal.h:96
ngram_funcs_s::free
void(* free)(ngram_model_t *model)
Implementation-specific function for freeing an ngram_model_t.
Definition: ngram_model_internal.h:114
ngram_funcs_s::apply_weights
int(* apply_weights)(ngram_model_t *model, float32 lw, float32 wip, float32 uw)
Implementation-specific function for applying language model weights.
Definition: ngram_model_internal.h:118
ngram_iter_s::m
int16 m
Order of history.
Definition: ngram_model_internal.h:196
gnode_s
A node in a generic list.
Definition: glist.h:100
ngram_model_s::writable
uint8 writable
Are word strings writable?
Definition: ngram_model_internal.h:63
ngram_model_s::n_counts
int32 * n_counts
Counts for 1, 2, 3, ...
Definition: ngram_model_internal.h:57
ngram_class_s::n_words
int32 n_words
Number of base words for this class.
Definition: ngram_model_internal.h:86
ngram_model_s::log_zero
int32 log_zero
Zero probability, cached here for quick lookup.
Definition: ngram_model_internal.h:72
ngram_model_s::refcount
int refcount
Reference count.
Definition: ngram_model_internal.h:56
ngram_model_s::n_1g_alloc
int32 n_1g_alloc
Number of allocated word strings (for new word addition)
Definition: ngram_model_internal.h:58
ngram_model_s::flags
uint8 flags
Any other flags we might care about (FIXME: Merge this and writable)
Definition: ngram_model_internal.h:64
hash_table_t
Definition: hash_table.h:159
ngram_funcs_s::raw_score
int32(* raw_score)(ngram_model_t *model, int32 wid, int32 *history, int32 n_hist, int32 *n_used)
Implementation-specific function for querying raw language model probability.
Definition: ngram_model_internal.h:134
ngram_model.h
N-Gram language models.
ngram_class_s::ngram_hash_s
Custom hash table for additional words.
Definition: ngram_model_internal.h:91
ngram_class_s::tag_wid
int32 tag_wid
Base word ID for this class tag.
Definition: ngram_model_internal.h:84
ngram_class_s::n_hash_inuse
int32 n_hash_inuse
Number of words in nword_hash.
Definition: ngram_model_internal.h:97
ngram_model_s::n
uint8 n
This is an n-gram model (1, 2, 3, ...).
Definition: ngram_model_internal.h:61
ngram_model_s::lmath
logmath_t * lmath
Log-math object.
Definition: ngram_model_internal.h:66
classdef_s
One class definition from a classdef file.
Definition: ngram_model_internal.h:203
ngram_model_s::log_uw
int32 log_uw
Log of unigram weight.
Definition: ngram_model_internal.h:69
ngram_class_s::start_wid
int32 start_wid
Starting base word ID for this class' words.
Definition: ngram_model_internal.h:85
ngram_funcs_s::add_ug
int32(* add_ug)(ngram_model_t *model, int32 wid, int32 lweight)
Implementation-specific function for adding unigrams.
Definition: ngram_model_internal.h:150
ngram_model_s::n_classes
uint8 n_classes
Number of classes (maximum 128)
Definition: ngram_model_internal.h:62
cmd_ln_t
Opaque structure used to hold the results of command-line parsing.
ngram_funcs_s::iter_free
void(* iter_free)(ngram_iter_t *itor)
Implementation-specific function for iterating.
Definition: ngram_model_internal.h:187
ngram_funcs_s
Implementation-specific functions for operating on ngram_model_t objects.
Definition: ngram_model_internal.h:110
ngram_model_s::lw
float32 lw
Language model scaling factor.
Definition: ngram_model_internal.h:67
ngram_iter_s
Base iterator structure for N-grams.
Definition: ngram_model_internal.h:193
ngram_class_s
Implementation of ngram_class_t.
Definition: ngram_model_internal.h:83
hash_table.h
Hash table implementation.
ngram_class_s::ngram_hash_s::prob1
int32 prob1
Probability for this word.
Definition: ngram_model_internal.h:93
ngram_iter_s::wids
int32 * wids
Scratch space for word IDs.
Definition: ngram_model_internal.h:195
ngram_funcs_s::flush
void(* flush)(ngram_model_t *model)
Implementation-specific function for purging N-Gram cache.
Definition: ngram_model_internal.h:155
ngram_model_s
Common implementation of ngram_model_t.
Definition: ngram_model_internal.h:55
ngram_class_s::ngram_hash_s::wid
int32 wid
Word ID of this bucket.
Definition: ngram_model_internal.h:92
ngram_funcs_s::score
int32(* score)(ngram_model_t *model, int32 wid, int32 *history, int32 n_hist, int32 *n_used)
Implementation-specific function for querying language model score.
Definition: ngram_model_internal.h:125
ngram_model_s::classes
struct ngram_class_s ** classes
Word class definitions.
Definition: ngram_model_internal.h:76
ngram_iter_s::successor
int16 successor
Is this a successor iterator?
Definition: ngram_model_internal.h:197
ngram_class_s::prob1
int32 * prob1
Probability table for base words.
Definition: ngram_model_internal.h:87
ngram_model_s::log_wip
int32 log_wip
Log of word insertion penalty.
Definition: ngram_model_internal.h:68
ngram_model_s::n_words
int32 n_words
Number of actual word strings (NOT the same as the number of unigrams, due to class words)...
Definition: ngram_model_internal.h:59