ngram_model_dmp.c
/* -*- c-basic-offset: 4; indent-tabs-mode: nil -*- */
/* ====================================================================
* Copyright (c) 1999-2007 Carnegie Mellon University. All rights
* reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
*
* This work was supported in part by funding from the Defense Advanced
* Research Projects Agency and the National Science Foundation of the
* United States of America, and the CMU Sphinx Speech Consortium.
*
* THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND
* ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
* THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY
* NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
* ====================================================================
*
*/
/*
* \file ngram_model_dmp.c DMP format language models
*
* Author: David Huggins-Daines <dhuggins@cs.cmu.edu>
*/
#include <assert.h>
#include <stdio.h>
#include <string.h>
#include <stdlib.h>
#include <limits.h>
#include "sphinxbase/ckd_alloc.h"
#include "sphinxbase/pio.h"
#include "sphinxbase/err.h"
#include "sphinxbase/byteorder.h"
#include "sphinxbase/listelem_alloc.h"
#include "ngram_model_dmp.h"
static const char darpa_hdr[] = "Darpa Trigram LM";
static ngram_funcs_t ngram_model_dmp_funcs;
#define TSEG_BASE(m,b) ((m)->lm3g.tseg_base[(b)>>LOG_BG_SEG_SZ])
#define FIRST_BG(m,u) ((m)->lm3g.unigrams[u].bigrams)
#define FIRST_TG(m,b) (TSEG_BASE((m),(b))+((m)->lm3g.bigrams[b].trigrams))
static unigram_t *
new_unigram_table(int32 n_ug)
{
unigram_t *table;
int32 i;
table = ckd_calloc(n_ug, sizeof(unigram_t));
for (i = 0; i < n_ug; i++) {
table[i].prob1.f = -99.0;
table[i].bo_wt1.f = -99.0;
}
return table;
}
ngram_model_t *
ngram_model_dmp_read(cmd_ln_t *config,
const char *file_name,
logmath_t *lmath)
{
ngram_model_t *base;
ngram_model_dmp_t *model;
FILE *fp;
int do_mmap, do_swap;
int32 is_pipe;
int32 i, j, k, vn, n, ts;
int32 n_unigram;
int32 n_bigram;
int32 n_trigram;
char str[1024];
unigram_t *ugptr;
bigram_t *bgptr;
trigram_t *tgptr;
char *tmp_word_str;
char *map_base = NULL;
size_t offset = 0;
base = NULL;
do_mmap = FALSE;
if (config)
do_mmap = cmd_ln_boolean_r(config, "-mmap");
if ((fp = fopen_comp(file_name, "rb", &is_pipe)) == NULL) {
E_ERROR("Dump file %s not found\n", file_name);
goto error_out;
}
if (is_pipe && do_mmap) {
E_WARN("Dump file is compressed, will not use memory-mapped I/O\n");
do_mmap = 0;
}
do_swap = FALSE;
if (fread(&k, sizeof(k), 1, fp) != 1)
goto error_out;
if (k != strlen(darpa_hdr)+1) {
SWAP_INT32(&k);
if (k != strlen(darpa_hdr)+1) {
E_ERROR("Wrong magic header size number %x: %s is not a dump file\n", k, file_name);
goto error_out;
}
do_swap = 1;
}
if (fread(str, 1, k, fp) != (size_t) k) {
E_ERROR("Cannot read header\n");
goto error_out;
}
if (strncmp(str, darpa_hdr, k) != 0) {
E_ERROR("Wrong header %s: %s is not a dump file\n", darpa_hdr);
goto error_out;
}
if (do_mmap) {
if (do_swap) {
E_INFO
("Byteswapping required, will not use memory-mapped I/O for LM file\n");
do_mmap = 0;
}
else {
E_INFO("Will use memory-mapped I/O for LM file\n");
#ifdef __ADSPBLACKFIN__ /* This is true for both VisualDSP++ and uClinux. */
E_FATAL("memory mapping is not supported at the moment.");
#else
#endif
}
}
if (fread(&k, sizeof(k), 1, fp) != 1)
goto error_out;
if (do_swap) SWAP_INT32(&k);
if (fread(str, 1, k, fp) != (size_t) k) {
E_ERROR("Cannot read LM filename in header\n");
goto error_out;
}
/* read version#, if present (must be <= 0) */
if (fread(&vn, sizeof(vn), 1, fp) != 1)
goto error_out;
if (do_swap) SWAP_INT32(&vn);
if (vn <= 0) {
/* read and don't compare timestamps (we don't care) */
if (fread(&ts, sizeof(ts), 1, fp) != 1)
goto error_out;
if (do_swap) SWAP_INT32(&ts);
/* read and skip format description */
for (;;) {
if (fread(&k, sizeof(k), 1, fp) != 1)
goto error_out;
if (do_swap) SWAP_INT32(&k);
if (k == 0)
break;
if (fread(str, 1, k, fp) != (size_t) k) {
E_ERROR("Failed to read word\n");
goto error_out;
}
}
/* read model->ucount */
if (fread(&n_unigram, sizeof(n_unigram), 1, fp) != 1)
goto error_out;
if (do_swap) SWAP_INT32(&n_unigram);
}
else {
n_unigram = vn;
}
/* read model->bcount, tcount */
if (fread(&n_bigram, sizeof(n_bigram), 1, fp) != 1)
goto error_out;
if (do_swap) SWAP_INT32(&n_bigram);
if (fread(&n_trigram, sizeof(n_trigram), 1, fp) != 1)
goto error_out;
if (do_swap) SWAP_INT32(&n_trigram);
E_INFO("ngrams 1=%d, 2=%d, 3=%d\n", n_unigram, n_bigram, n_trigram);
/* Allocate space for LM, including initial OOVs and placeholders; initialize it */
model = ckd_calloc(1, sizeof(*model));
base = &model->base;
if (n_trigram > 0)
n = 3;
else if (n_bigram > 0)
n = 2;
else
n = 1;
ngram_model_init(base, &ngram_model_dmp_funcs, lmath, n, n_unigram);
base->n_counts[0] = n_unigram;
base->n_counts[1] = n_bigram;
base->n_counts[2] = n_trigram;
/* read unigrams (always in memory, as they contain dictionary
* mappings that can't be precomputed, and also could have OOVs added) */
model->lm3g.unigrams = new_unigram_table(n_unigram + 1);
ugptr = model->lm3g.unigrams;
for (i = 0; i <= n_unigram; ++i) {
/* Skip over the mapping ID, we don't care about it. */
if (fread(ugptr, sizeof(int32), 1, fp) != 1) {
E_ERROR("Failed to read maping id %d\n", i);
goto error_out;
}
/* Read the actual unigram structure. */
if (fread(ugptr, sizeof(unigram_t), 1, fp) != 1) {
E_ERROR("Failed to read unigrams data\n");
ngram_model_free(base);
fclose_comp(fp, is_pipe);
return NULL;
}
/* Byte swap if necessary. */
if (do_swap) {
SWAP_INT32(&ugptr->prob1.l);
SWAP_INT32(&ugptr->bo_wt1.l);
SWAP_INT32(&ugptr->bigrams);
}
/* Convert values to log. */
ugptr->prob1.l = logmath_log10_to_log(lmath, ugptr->prob1.f);
ugptr->bo_wt1.l = logmath_log10_to_log(lmath, ugptr->bo_wt1.f);
E_DEBUG(2, ("ug %d: prob %d bo %d bigrams %d\n",
i, ugptr->prob1.l, ugptr->bo_wt1.l, ugptr->bigrams));
++ugptr;
}
E_INFO("%8d = LM.unigrams(+trailer) read\n", n_unigram);
/* Now mmap() the file and read in the rest of the (read-only) stuff. */
if (do_mmap) {
offset = ftell(fp);
/* Check for improper word alignment. */
if (offset & 0x3) {
E_WARN("-mmap specified, but trigram index is not word-aligned. Will not memory-map.\n");
do_mmap = FALSE;
}
else {
model->dump_mmap = mmio_file_read(file_name);
if (model->dump_mmap == NULL) {
do_mmap = FALSE;
}
else {
map_base = mmio_file_ptr(model->dump_mmap);
}
}
}
if (n_bigram > 0) {
/* read bigrams */
if (do_mmap) {
model->lm3g.bigrams = (bigram_t *) (map_base + offset);
offset += (n_bigram + 1) * sizeof(bigram_t);
}
else {
model->lm3g.bigrams =
ckd_calloc(n_bigram + 1, sizeof(bigram_t));
if (fread(model->lm3g.bigrams, sizeof(bigram_t), n_bigram + 1, fp)
!= (size_t) n_bigram + 1) {
E_ERROR("Failed to read bigrams data\n");
goto error_out;
}
if (do_swap) {
for (i = 0, bgptr = model->lm3g.bigrams; i <= n_bigram;
i++, bgptr++) {
SWAP_INT16(&bgptr->wid);
SWAP_INT16(&bgptr->prob2);
SWAP_INT16(&bgptr->bo_wt2);
SWAP_INT16(&bgptr->trigrams);
}
}
}
E_INFO("%8d = LM.bigrams(+trailer) read\n", n_bigram);
}
/* read trigrams */
if (n_trigram > 0) {
if (do_mmap) {
model->lm3g.trigrams = (trigram_t *) (map_base + offset);
offset += n_trigram * sizeof(trigram_t);
}
else {
model->lm3g.trigrams =
ckd_calloc(n_trigram, sizeof(trigram_t));
if (fread
(model->lm3g.trigrams, sizeof(trigram_t), n_trigram, fp)
!= (size_t) n_trigram) {
E_ERROR("Failed to read trigrams data\n");
goto error_out;
}
if (do_swap) {
for (i = 0, tgptr = model->lm3g.trigrams; i < n_trigram;
i++, tgptr++) {
SWAP_INT16(&tgptr->wid);
SWAP_INT16(&tgptr->prob3);
}
}
}
E_INFO("%8d = LM.trigrams read\n", n_trigram);
/* Initialize tginfo */
model->lm3g.tginfo = ckd_calloc(n_unigram, sizeof(tginfo_t *));
model->lm3g.le = listelem_alloc_init(sizeof(tginfo_t));
}
if (n_bigram > 0) {
/* read n_prob2 and prob2 array (in memory) */
if (do_mmap)
fseek(fp, offset, SEEK_SET);
if (fread(&k, sizeof(k), 1, fp) != 1)
goto error_out;
if (do_swap) SWAP_INT32(&k);
model->lm3g.n_prob2 = k;
model->lm3g.prob2 = ckd_calloc(k, sizeof(*model->lm3g.prob2));
if (fread(model->lm3g.prob2, sizeof(*model->lm3g.prob2), k, fp) != (size_t) k) {
E_ERROR("fread(prob2) failed\n");
goto error_out;
}
for (i = 0; i < k; i++) {
if (do_swap)
SWAP_INT32(&model->lm3g.prob2[i].l);
/* Convert values to log. */
model->lm3g.prob2[i].l = logmath_log10_to_log(lmath, model->lm3g.prob2[i].f);
}
E_INFO("%8d = LM.prob2 entries read\n", k);
}
/* read n_bo_wt2 and bo_wt2 array (in memory) */
if (base->n > 2) {
if (fread(&k, sizeof(k), 1, fp) != 1)
goto error_out;
if (do_swap) SWAP_INT32(&k);
model->lm3g.n_bo_wt2 = k;
model->lm3g.bo_wt2 = ckd_calloc(k, sizeof(*model->lm3g.bo_wt2));
if (fread(model->lm3g.bo_wt2, sizeof(*model->lm3g.bo_wt2), k, fp) != (size_t) k) {
E_ERROR("Failed to read backoff weights\n");
goto error_out;
}
for (i = 0; i < k; i++) {
if (do_swap)
SWAP_INT32(&model->lm3g.bo_wt2[i].l);
/* Convert values to log. */
model->lm3g.bo_wt2[i].l = logmath_log10_to_log(lmath, model->lm3g.bo_wt2[i].f);
}
E_INFO("%8d = LM.bo_wt2 entries read\n", k);
}
/* read n_prob3 and prob3 array (in memory) */
if (base->n > 2) {
if (fread(&k, sizeof(k), 1, fp) != 1)
goto error_out;
if (do_swap) SWAP_INT32(&k);
model->lm3g.n_prob3 = k;
model->lm3g.prob3 = ckd_calloc(k, sizeof(*model->lm3g.prob3));
if (fread(model->lm3g.prob3, sizeof(*model->lm3g.prob3), k, fp) != (size_t) k) {
E_ERROR("Failed to read trigram probability\n");
goto error_out;
}
for (i = 0; i < k; i++) {
if (do_swap)
SWAP_INT32(&model->lm3g.prob3[i].l);
/* Convert values to log. */
model->lm3g.prob3[i].l = logmath_log10_to_log(lmath, model->lm3g.prob3[i].f);
}
E_INFO("%8d = LM.prob3 entries read\n", k);
}
/* read tseg_base size and tseg_base */
if (do_mmap)
offset = ftell(fp);
if (n_trigram > 0) {
if (do_mmap) {
memcpy(&k, map_base + offset, sizeof(k));
offset += sizeof(int32);
model->lm3g.tseg_base = (int32 *) (map_base + offset);
offset += k * sizeof(int32);
}
else {
k = (n_bigram + 1) / BG_SEG_SZ + 1;
if (fread(&k, sizeof(k), 1, fp) != 1)
goto error_out;
if (do_swap) SWAP_INT32(&k);
model->lm3g.tseg_base = ckd_calloc(k, sizeof(int32));
if (fread(model->lm3g.tseg_base, sizeof(int32), k, fp) !=
(size_t) k) {
E_ERROR("Failed to read trigram index\n");
goto error_out;
}
if (do_swap)
for (i = 0; i < k; i++)
SWAP_INT32(&model->lm3g.tseg_base[i]);
}
E_INFO("%8d = LM.tseg_base entries read\n", k);
}
/* read ascii word strings */
if (do_mmap) {
memcpy(&k, map_base + offset, sizeof(k));
offset += sizeof(int32);
tmp_word_str = (char *) (map_base + offset);
offset += k;
}
else {
base->writable = TRUE;
if (fread(&k, sizeof(k), 1, fp) != 1)
goto error_out;
if (do_swap) SWAP_INT32(&k);
tmp_word_str = ckd_calloc(k, 1);
if (fread(tmp_word_str, 1, k, fp) != (size_t) k) {
E_ERROR("Failed to read words\n");
goto error_out;
}
}
/* First make sure string just read contains n_counts[0] words (PARANOIA!!) */
for (i = 0, j = 0; i < k; i++)
if (tmp_word_str[i] == '\0')
j++;
if (j != n_unigram) {
E_ERROR("Error reading word strings (%d doesn't match n_unigrams %d)\n",
j, n_unigram);
goto error_out;
}
/* Break up string just read into words */
if (do_mmap) {
j = 0;
for (i = 0; i < n_unigram; i++) {
base->word_str[i] = tmp_word_str + j;
if (hash_table_enter(base->wid, base->word_str[i],
(void *)(long)i) != (void *)(long)i) {
E_WARN("Duplicate word in dictionary: %s\n", base->word_str[i]);
}
j += strlen(base->word_str[i]) + 1;
}
}
else {
j = 0;
for (i = 0; i < n_unigram; i++) {
base->word_str[i] = ckd_salloc(tmp_word_str + j);
if (hash_table_enter(base->wid, base->word_str[i],
(void *)(long)i) != (void *)(long)i) {
E_WARN("Duplicate word in dictionary: %s\n", base->word_str[i]);
}
j += strlen(base->word_str[i]) + 1;
}
free(tmp_word_str);
}
E_INFO("%8d = ascii word strings read\n", i);
fclose_comp(fp, is_pipe);
return base;
error_out:
if (fp)
fclose_comp(fp, is_pipe);
ngram_model_free(base);
return NULL;
}
ngram_model_dmp_t *
ngram_model_dmp_build(ngram_model_t *base)
{
ngram_model_dmp_t *model;
ngram_model_t *newbase;
ngram_iter_t *itor;
sorted_list_t sorted_prob2;
sorted_list_t sorted_bo_wt2;
sorted_list_t sorted_prob3;
bigram_t *bgptr;
trigram_t *tgptr;
int i, bgcount, tgcount, seg;
if (base->funcs == &ngram_model_dmp_funcs) {
E_INFO("Using existing DMP model.\n");
return (ngram_model_dmp_t *)ngram_model_retain(base);
}
/* Initialize new base model structure with params from base. */
E_INFO("Building DMP model...\n");
model = ckd_calloc(1, sizeof(*model));
newbase = &model->base;
ngram_model_init(newbase, &ngram_model_dmp_funcs,
logmath_retain(base->lmath),
base->n, base->n_counts[0]);
/* Copy N-gram counts over. */
memcpy(newbase->n_counts, base->n_counts,
base->n * sizeof(*base->n_counts));
/* Make sure word strings are freed. */
newbase->writable = TRUE;
/* Initialize unigram table and string table. */
model->lm3g.unigrams = new_unigram_table(newbase->n_counts[0] + 1);
for (itor = ngram_model_mgrams(base, 0); itor;
itor = ngram_iter_next(itor)) {
int32 prob1, bo_wt1;
int32 const *wids;
/* Can't guarantee they will go in unigram order, so just to
* be correct, we do this... */
wids = ngram_iter_get(itor, &prob1, &bo_wt1);
model->lm3g.unigrams[wids[0]].prob1.l = prob1;
model->lm3g.unigrams[wids[0]].bo_wt1.l = bo_wt1;
newbase->word_str[wids[0]] = ckd_salloc(ngram_word(base, wids[0]));
if ((hash_table_enter_int32(newbase->wid,
newbase->word_str[wids[0]], wids[0]))
!= wids[0]) {
E_WARN("Duplicate word in dictionary: %s\n", newbase->word_str[wids[0]]);
}
}
E_INFO("%8d = #unigrams created\n", newbase->n_counts[0]);
if (newbase->n < 2)
return model;
/* Construct quantized probability table for bigrams and
* (optionally) trigrams. Hesitate to use the "sorted list" thing
* since it isn't so useful, but it's there already. */
init_sorted_list(&sorted_prob2);
if (newbase->n > 2) {
init_sorted_list(&sorted_bo_wt2);
init_sorted_list(&sorted_prob3);
}
/* Construct bigram and trigram arrays. */
bgptr = model->lm3g.bigrams = ckd_calloc(newbase->n_counts[1] + 1, sizeof(bigram_t));
if (newbase->n > 2) {
tgptr = model->lm3g.trigrams = ckd_calloc(newbase->n_counts[2], sizeof(trigram_t));
model->lm3g.tseg_base =
ckd_calloc((newbase->n_counts[1] + 1) / BG_SEG_SZ + 1, sizeof(int32));
}
else
tgptr = NULL;
/* Since bigrams and trigrams have to be contiguous with others
* with the same N-1-gram, we traverse them in depth-first order
* to build the bigram and trigram arrays. */
for (i = 0; i < newbase->n_counts[0]; ++i) {
ngram_iter_t *uitor;
bgcount = bgptr - model->lm3g.bigrams;
/* First bigram index (same as next if no bigrams...) */
model->lm3g.unigrams[i].bigrams = bgcount;
E_DEBUG(2, ("unigram %d: %s => bigram %d\n", i, newbase->word_str[i], bgcount));
/* All bigrams corresponding to unigram i */
uitor = ngram_ng_iter(base, i, NULL, 0);
for (itor = ngram_iter_successors(uitor);
itor; ++bgptr, itor = ngram_iter_next(itor)) {
int32 prob2, bo_wt2;
int32 const *wids;
ngram_iter_t *titor;
wids = ngram_iter_get(itor, &prob2, &bo_wt2);
assert (bgptr - model->lm3g.bigrams < newbase->n_counts[1]);
bgptr->wid = wids[1];
bgptr->prob2 = sorted_id(&sorted_prob2, &prob2);
if (newbase->n > 2) {
tgcount = (tgptr - model->lm3g.trigrams);
bgcount = (bgptr - model->lm3g.bigrams);
/* Backoff weight (only if there are trigrams...) */
bgptr->bo_wt2 = sorted_id(&sorted_bo_wt2, &bo_wt2);
/* Find bigram segment for this bigram (this isn't
* used unless there are trigrams) */
seg = bgcount >> LOG_BG_SEG_SZ;
/* If we just crossed a bigram segment boundary, then
* point tseg_base for the new segment to the current
* trigram pointer. */
if (seg != (bgcount - 1) >> LOG_BG_SEG_SZ)
model->lm3g.tseg_base[seg] = tgcount;
/* Now calculate the trigram offset. */
bgptr->trigrams = tgcount - model->lm3g.tseg_base[seg];
E_DEBUG(2, ("bigram %d %s %s => trigram %d:%d\n",
bgcount,
newbase->word_str[wids[0]],
newbase->word_str[wids[1]],
seg, bgptr->trigrams));
/* And fill in successors' trigram info. */
for (titor = ngram_iter_successors(itor);
titor; ++tgptr, titor = ngram_iter_next(titor)) {
int32 prob3, dummy;
assert(tgptr - model->lm3g.trigrams < newbase->n_counts[2]);
wids = ngram_iter_get(titor, &prob3, &dummy);
tgptr->wid = wids[2];
tgptr->prob3 = sorted_id(&sorted_prob3, &prob3);
E_DEBUG(2, ("trigram %d %s %s %s => prob %d\n",
tgcount,
newbase->word_str[wids[0]],
newbase->word_str[wids[1]],
newbase->word_str[wids[2]],
tgptr->prob3));
}
}
}
ngram_iter_free(uitor);
}
/* Add sentinal unigram and bigram records. */
bgcount = bgptr - model->lm3g.bigrams;
tgcount = tgptr - model->lm3g.trigrams;
seg = bgcount >> LOG_BG_SEG_SZ;
if (seg != (bgcount - 1) >> LOG_BG_SEG_SZ)
model->lm3g.tseg_base[seg] = tgcount;
model->lm3g.unigrams[i].bigrams = bgcount;
if (newbase->n > 2)
bgptr->trigrams = tgcount - model->lm3g.tseg_base[seg];
/* Now create probability tables. */
model->lm3g.n_prob2 = sorted_prob2.free;
model->lm3g.prob2 = vals_in_sorted_list(&sorted_prob2);
E_INFO("%8d = #bigrams created\n", newbase->n_counts[1]);
E_INFO("%8d = #prob2 entries\n", model->lm3g.n_prob2);
free_sorted_list(&sorted_prob2);
if (newbase->n > 2) {
/* Create trigram bo-wts array. */
model->lm3g.n_bo_wt2 = sorted_bo_wt2.free;
model->lm3g.bo_wt2 = vals_in_sorted_list(&sorted_bo_wt2);
free_sorted_list(&sorted_bo_wt2);
E_INFO("%8d = #bo_wt2 entries\n", model->lm3g.n_bo_wt2);
/* Create trigram probability table. */
model->lm3g.n_prob3 = sorted_prob3.free;
model->lm3g.prob3 = vals_in_sorted_list(&sorted_prob3);
E_INFO("%8d = #trigrams created\n", newbase->n_counts[2]);
E_INFO("%8d = #prob3 entries\n", model->lm3g.n_prob3);
free_sorted_list(&sorted_prob3);
/* Initialize tginfo */
model->lm3g.tginfo = ckd_calloc(newbase->n_counts[0], sizeof(tginfo_t *));
model->lm3g.le = listelem_alloc_init(sizeof(tginfo_t));
}
return model;
}
static void
fwrite_int32(FILE *fh, int32 val)
{
fwrite(&val, 4, 1, fh);
}
static void
fwrite_ug(FILE *fh, unigram_t *ug, logmath_t *lmath)
{
int32 bogus = -1;
float32 log10val;
/* Bogus dictionary mapping field. */
fwrite(&bogus, 4, 1, fh);
/* Convert values to log10. */
log10val = logmath_log_to_log10(lmath, ug->prob1.l);
fwrite(&log10val, 4, 1, fh);
log10val = logmath_log_to_log10(lmath, ug->bo_wt1.l);
fwrite(&log10val, 4, 1, fh);
fwrite_int32(fh, ug->bigrams);
}
static void
fwrite_bg(FILE *fh, bigram_t *bg)
{
fwrite(bg, sizeof(*bg), 1, fh);
}
static void
fwrite_tg(FILE *fh, trigram_t *tg)
{
fwrite(tg, sizeof(*tg), 1, fh);
}
static char const *fmtdesc[] = {
"BEGIN FILE FORMAT DESCRIPTION",
"Header string length (int32) and string (including trailing 0)",
"Original LM filename string-length (int32) and filename (including trailing 0)",
"(int32) version number (present iff value <= 0)",
"(int32) original LM file modification timestamp (iff version# present)",
"(int32) string-length and string (including trailing 0) (iff version# present)",
"... previous entry continued any number of times (iff version# present)",
"(int32) 0 (terminating sequence of strings) (iff version# present)",
"(int32) log_bg_seg_sz (present iff different from default value of LOG2_BG_SEG_SZ)",
"(int32) lm_t.ucount (must be > 0)",
"(int32) lm_t.bcount",
"(int32) lm_t.tcount",
"lm_t.ucount+1 unigrams (including sentinel)",
"lm_t.bcount+1 bigrams (including sentinel 64 bits (bg_t) each if version=-1/-2, 128 bits (bg32_t) each if version=-3",
"lm_t.tcount trigrams (present iff lm_t.tcount > 0 32 bits (tg_t) each if version=-1/-2, 64 bits (tg32_t) each if version=-3)",
"(int32) lm_t.n_prob2",
"(int32) lm_t.prob2[]",
"(int32) lm_t.n_bo_wt2 (present iff lm_t.tcount > 0)",
"(int32) lm_t.bo_wt2[] (present iff lm_t.tcount > 0)",
"(int32) lm_t.n_prob3 (present iff lm_t.tcount > 0)",
"(int32) lm_t.prob3[] (present iff lm_t.tcount > 0)",
"(int32) (lm_t.bcount+1)/BG_SEG_SZ+1 (present iff lm_t.tcount > 0)",
"(int32) lm_t.tseg_base[] (present iff lm_t.tcount > 0)",
"(int32) Sum(all word string-lengths, including trailing 0 for each)",
"All word strings (including trailing 0 for each)",
"END FILE FORMAT DESCRIPTION",
NULL,
};
static void
ngram_model_dmp_write_header(FILE * fh)
{
int32 k;
k = strlen(darpa_hdr) + 1;
fwrite_int32(fh, k);
fwrite(darpa_hdr, 1, k, fh);
}
static void
ngram_model_dmp_write_lm_filename(FILE * fh, const char *lmfile)
{
int32 k;
k = strlen(lmfile) + 1;
fwrite_int32(fh, k);
fwrite(lmfile, 1, k, fh);
}
#define LMDMP_VERSION_TG_16BIT -1
static void
ngram_model_dmp_write_version(FILE * fh, int32 mtime)
{
fwrite_int32(fh, LMDMP_VERSION_TG_16BIT); /* version # */
fwrite_int32(fh, mtime);
}
static void
ngram_model_dmp_write_ngram_counts(FILE * fh, ngram_model_t *model)
{
fwrite_int32(fh, model->n_counts[0]);
fwrite_int32(fh, model->n_counts[1]);
fwrite_int32(fh, model->n_counts[2]);
}
static void
ngram_model_dmp_write_fmtdesc(FILE * fh)
{
int32 i, k;
long pos;
/* Write file format description into header */
for (i = 0; fmtdesc[i] != NULL; i++) {
k = strlen(fmtdesc[i]) + 1;
fwrite_int32(fh, k);
fwrite(fmtdesc[i], 1, k, fh);
}
/* Pad it out in order to achieve 32-bit alignment */
pos = ftell(fh);
k = pos & 3;
if (k) {
fwrite_int32(fh, 4-k);
fwrite("!!!!", 1, 4-k, fh);
}
fwrite_int32(fh, 0);
}
static void
ngram_model_dmp_write_unigram(FILE *fh, ngram_model_t *model)
{
ngram_model_dmp_t *lm = (ngram_model_dmp_t *)model;
int32 i;
for (i = 0; i <= model->n_counts[0]; i++) {
fwrite_ug(fh, &(lm->lm3g.unigrams[i]), model->lmath);
}
}
static void
ngram_model_dmp_write_bigram(FILE *fh, ngram_model_t *model)
{
ngram_model_dmp_t *lm = (ngram_model_dmp_t *)model;
int32 i;
for (i = 0; i <= model->n_counts[1]; i++) {
fwrite_bg(fh, &(lm->lm3g.bigrams[i]));
}
}
static void
ngram_model_dmp_write_trigram(FILE *fh, ngram_model_t *model)
{
ngram_model_dmp_t *lm = (ngram_model_dmp_t *)model;
int32 i;
for (i = 0; i < model->n_counts[2]; i++) {
fwrite_tg(fh, &(lm->lm3g.trigrams[i]));
}
}
static void
ngram_model_dmp_write_bgprob(FILE *fh, ngram_model_t *model)
{
ngram_model_dmp_t *lm = (ngram_model_dmp_t *)model;
int32 i;
fwrite_int32(fh, lm->lm3g.n_prob2);
for (i = 0; i < lm->lm3g.n_prob2; i++) {
float32 log10val = logmath_log_to_log10(model->lmath, lm->lm3g.prob2[i].l);
fwrite(&log10val, 4, 1, fh);
}
}
static void
ngram_model_dmp_write_tgbowt(FILE *fh, ngram_model_t *model)
{
ngram_model_dmp_t *lm = (ngram_model_dmp_t *)model;
int32 i;
fwrite_int32(fh, lm->lm3g.n_bo_wt2);
for (i = 0; i < lm->lm3g.n_bo_wt2; i++) {
float32 log10val = logmath_log_to_log10(model->lmath, lm->lm3g.bo_wt2[i].l);
fwrite(&log10val, 4, 1, fh);
}
}
static void
ngram_model_dmp_write_tgprob(FILE *fh, ngram_model_t *model)
{
ngram_model_dmp_t *lm = (ngram_model_dmp_t *)model;
int32 i;
fwrite_int32(fh, lm->lm3g.n_prob3);
for (i = 0; i < lm->lm3g.n_prob3; i++) {
float32 log10val = logmath_log_to_log10(model->lmath, lm->lm3g.prob3[i].l);
fwrite(&log10val, 4, 1, fh);
}
}
static void
ngram_model_dmp_write_tg_segbase(FILE *fh, ngram_model_t *model)
{
ngram_model_dmp_t *lm = (ngram_model_dmp_t *)model;
int32 i, k;
k = (model->n_counts[1] + 1) / BG_SEG_SZ + 1;
fwrite_int32(fh, k);
for (i = 0; i < k; i++)
fwrite_int32(fh, lm->lm3g.tseg_base[i]);
}
static void
ngram_model_dmp_write_wordstr(FILE *fh, ngram_model_t *model)
{
int32 i, k;
k = 0;
for (i = 0; i < model->n_counts[0]; i++)
k += strlen(model->word_str[i]) + 1;
fwrite_int32(fh, k);
for (i = 0; i < model->n_counts[0]; i++)
fwrite(model->word_str[i], 1,
strlen(model->word_str[i]) + 1, fh);
}
int
ngram_model_dmp_write(ngram_model_t *base,
const char *file_name)
{
ngram_model_dmp_t *model;
ngram_model_t *newbase;
FILE *fh;
/* First, construct a DMP model from the base model. */
model = ngram_model_dmp_build(base);
newbase = &model->base;
/* Now write it, confident in the knowledge that it's the right
* kind of language model internally. */
if ((fh = fopen(file_name, "wb")) == NULL) {
E_ERROR("Cannot create file %s\n", file_name);
return -1;
}
ngram_model_dmp_write_header(fh);
ngram_model_dmp_write_lm_filename(fh, file_name);
ngram_model_dmp_write_version(fh, 0);
ngram_model_dmp_write_fmtdesc(fh);
ngram_model_dmp_write_ngram_counts(fh, newbase);
ngram_model_dmp_write_unigram(fh, newbase);
if (newbase->n > 1) {
ngram_model_dmp_write_bigram(fh, newbase);
if (newbase->n > 2) {
ngram_model_dmp_write_trigram(fh, newbase);
}
ngram_model_dmp_write_bgprob(fh, newbase);
if (newbase->n > 2) {
ngram_model_dmp_write_tgbowt(fh, newbase);
ngram_model_dmp_write_tgprob(fh, newbase);
ngram_model_dmp_write_tg_segbase(fh, newbase);
}
}
ngram_model_dmp_write_wordstr(fh, newbase);
ngram_model_free(newbase);
return fclose(fh);
}
static int
ngram_model_dmp_apply_weights(ngram_model_t *base, float32 lw,
float32 wip, float32 uw)
{
ngram_model_dmp_t *model = (ngram_model_dmp_t *)base;
lm3g_apply_weights(base, &model->lm3g, lw, wip, uw);
return 0;
}
/* Lousy "templating" for things that are largely the same in DMP and
* ARPA models, except for the bigram and trigram types and some
* names. */
#define NGRAM_MODEL_TYPE ngram_model_dmp_t
#include "lm3g_templates.c"
static void
ngram_model_dmp_free(ngram_model_t *base)
{
ngram_model_dmp_t *model = (ngram_model_dmp_t *)base;
ckd_free(model->lm3g.unigrams);
ckd_free(model->lm3g.prob2);
if (model->dump_mmap) {
mmio_file_unmap(model->dump_mmap);
}
else {
ckd_free(model->lm3g.bigrams);
if (base->n > 2) {
ckd_free(model->lm3g.trigrams);
ckd_free(model->lm3g.tseg_base);
}
}
if (base->n > 2) {
ckd_free(model->lm3g.bo_wt2);
ckd_free(model->lm3g.prob3);
}
lm3g_tginfo_free(base, &model->lm3g);
}
static ngram_funcs_t ngram_model_dmp_funcs = {
ngram_model_dmp_free, /* free */
ngram_model_dmp_apply_weights, /* apply_weights */
lm3g_template_score, /* score */
lm3g_template_raw_score, /* raw_score */
lm3g_template_add_ug, /* add_ug */
lm3g_template_flush, /* flush */
lm3g_template_iter, /* iter */
lm3g_template_mgrams, /* mgrams */
lm3g_template_successors, /* successors */
lm3g_template_iter_get, /* iter_get */
lm3g_template_iter_next, /* iter_next */
lm3g_template_iter_free /* iter_free */
};
unigram_s::bo_wt1
lmprob_t bo_wt1
Unigram backoff weight.
Definition: lm3g_model.h:93
lm3g_model_s::le
listelem_alloc_t * le
List element allocator for tginfo.
Definition: lm3g_model.h:156
ngram_model_s::funcs
struct ngram_funcs_s * funcs
Implementation-specific methods.
Definition: ngram_model_internal.h:77
ckd_calloc
#define ckd_calloc(n, sz)
Macros to simplify the use of above functions.
Definition: ckd_alloc.h:248
lm3g_model_s::n_prob3
int32 n_prob3
prob3 size
Definition: lm3g_model.h:151
hash_table_enter_int32
#define hash_table_enter_int32(h, k, v)
Add a 32-bit integer value to a hash table.
Definition: hash_table.h:228
ngram_model_s::wid
hash_table_t * wid
Mapping of unigram names to word IDs.
Definition: ngram_model_internal.h:74
ngram_model_s::word_str
char ** word_str
Unigram names.
Definition: ngram_model_internal.h:73
ngram_iter_free
SPHINXBASE_EXPORT void ngram_iter_free(ngram_iter_t *itor)
Terminate an M-gram iterator.
Definition: ngram_model.c:764
E_DEBUG
#define E_DEBUG(level, x)
Print debugging information to standard error stream.
Definition: err.h:212
E_INFO
#define E_INFO
Print logging information to standard error stream.
Definition: err.h:147
ckd_alloc.h
Sphinx's memory allocation/deallocation routines.
ngram_iter_get
SPHINXBASE_EXPORT int32 const * ngram_iter_get(ngram_iter_t *itor, int32 *out_score, int32 *out_bowt)
Get information from the current M-gram in an iterator.
Definition: ngram_model.c:750
logmath_s
Definition: logmath.c:49
lm3g_model_s::prob2
lmprob_t * prob2
Table of actual bigram probs.
Definition: lm3g_model.h:146
bigram_s
Bigram structure.
Definition: ngram_model_arpa.h:52
mmio_file_unmap
SPHINXBASE_EXPORT void mmio_file_unmap(mmio_file_t *mf)
Unmap a file, releasing memory associated with it.
Definition: mmio.c:240
ngram_iter_next
SPHINXBASE_EXPORT ngram_iter_t * ngram_iter_next(ngram_iter_t *itor)
Advance an M-gram iterator.
Definition: ngram_model.c:758
ngram_model_s::writable
uint8 writable
Are word strings writable?
Definition: ngram_model_internal.h:63
ngram_model_s::n_counts
int32 * n_counts
Counts for 1, 2, 3, ...
Definition: ngram_model_internal.h:57
ngram_model_free
SPHINXBASE_EXPORT int ngram_model_free(ngram_model_t *model)
Release memory associated with an N-Gram model.
Definition: ngram_model.c:254
ckd_salloc
#define ckd_salloc(ptr)
Macro for ckd_salloc
Definition: ckd_alloc.h:264
trigram_s::prob3
uint16 prob3
Index into array of actual trigram probs.
Definition: ngram_model_arpa.h:68
ngram_iter_successors
SPHINXBASE_EXPORT ngram_iter_t * ngram_iter_successors(ngram_iter_t *itor)
Iterate over all M-gram successors of an M-1-gram.
Definition: ngram_model.c:741
ngram_model_dmp_s
Subclass of ngram_model for DMP file reading.
Definition: ngram_model_dmp.h:76
ckd_free
SPHINXBASE_EXPORT void ckd_free(void *ptr)
Test and free a 1-D array.
Definition: ckd_alloc.c:241
E_WARN
#define E_WARN
Print warning information to standard error stream.
Definition: err.h:164
ngram_model_dmp_s::base
ngram_model_t base
Base ngram_model_t structure.
Definition: ngram_model_dmp.h:77
unigram_s
Unigram structure (common among all lm3g implementations)
Definition: lm3g_model.h:91
lm3g_model_s::n_bo_wt2
int32 n_bo_wt2
bo_wt2 size
Definition: lm3g_model.h:149
unigram_s::bigrams
int32 bigrams
Index of 1st entry in lm_t.bigrams[].
Definition: lm3g_model.h:94
lm3g_model_s::prob3
lmprob_t * prob3
Table of actual trigram probs.
Definition: lm3g_model.h:150
mmio_file_ptr
SPHINXBASE_EXPORT void * mmio_file_ptr(mmio_file_t *mf)
Get a pointer to the memory mapped for a file.
Definition: mmio.c:251
tginfo_s
Trigram information cache.
Definition: lm3g_model.h:129
trigram_s
Trigram structure.
Definition: ngram_model_arpa.h:66
logmath_retain
SPHINXBASE_EXPORT logmath_t * logmath_retain(logmath_t *lmath)
Retain ownership of a log table.
Definition: logmath.c:335
ngram_model_retain
SPHINXBASE_EXPORT ngram_model_t * ngram_model_retain(ngram_model_t *model)
Retain ownership of an N-Gram model.
Definition: ngram_model.c:239
listelem_alloc.h
Fast memory allocator for uniformly sized objects.
bigram_s::trigrams
uint16 trigrams
Index of 1st entry in lm_t.trigrams[], RELATIVE TO its segment base (see above)
Definition: ngram_model_arpa.h:56
ngram_model_s::n
uint8 n
This is an n-gram model (1, 2, 3, ...).
Definition: ngram_model_internal.h:61
bigram_s::prob2
uint16 prob2
Index into array of actual bigram probs.
Definition: ngram_model_arpa.h:54
err.h
Implementation of logging routines.
sorted_list_t
The sorted list.
Definition: lm3g_model.h:82
ngram_model_s::lmath
logmath_t * lmath
Log-math object.
Definition: ngram_model_internal.h:66
bigram_s::bo_wt2
uint16 bo_wt2
Index into array of actual bigram backoff wts.
Definition: ngram_model_arpa.h:55
hash_table_enter
SPHINXBASE_EXPORT void * hash_table_enter(hash_table_t *h, const char *key, void *val)
Try to add a new entry with given key and associated value to hash table h.
Definition: hash_table.c:508
fopen_comp
SPHINXBASE_EXPORT FILE * fopen_comp(const char *file, const char *mode, int32 *ispipe)
Like fopen, but use popen and zcat if it is determined that "file" is compressed (i.e., has a .z, .Z, .gz, or .GZ extension).
Definition: pio.c:98
ngram_model_dmp_s::dump_mmap
mmio_file_t * dump_mmap
mmap() of dump file (or NULL if none)
Definition: ngram_model_dmp.h:79
ngram_ng_iter
SPHINXBASE_EXPORT ngram_iter_t * ngram_ng_iter(ngram_model_t *model, int32 wid, int32 *history, int32 n_hist)
Get an iterator over M-grams pointing to the specified M-gram.
Definition: ngram_model.c:731
unigram_s::prob1
lmprob_t prob1
Unigram probability.
Definition: lm3g_model.h:92
cmd_ln_t
Opaque structure used to hold the results of command-line parsing.
E_FATAL
#define E_FATAL
Exit with non-zero status after error message.
Definition: err.h:127
E_ERROR
#define E_ERROR
Print error message to standard error stream.
Definition: err.h:169
lm3g_model_s::bo_wt2
lmprob_t * bo_wt2
Table of actual bigram backoff weights.
Definition: lm3g_model.h:148
ngram_funcs_s
Implementation-specific functions for operating on ngram_model_t objects.
Definition: ngram_model_internal.h:110
listelem_alloc_init
SPHINXBASE_EXPORT listelem_alloc_t * listelem_alloc_init(size_t elemsize)
Initialize and return a list element allocator.
Definition: listelem_alloc.c:87
ngram_iter_s
Base iterator structure for N-grams.
Definition: ngram_model_internal.h:193
cmd_ln_boolean_r
#define cmd_ln_boolean_r(c, n)
Retrieve a boolean value from a command-line object.
Definition: cmd_ln.h:334
logmath_log_to_log10
SPHINXBASE_EXPORT float64 logmath_log_to_log10(logmath_t *lmath, int logb_p)
Convert integer log in base B to base 10 log (in floating point).
Definition: logmath.c:480
ngram_model_dmp_s::lm3g
lm3g_model_t lm3g
Common lm3g_model_t structure.
Definition: ngram_model_dmp.h:78
ngram_model_s
Common implementation of ngram_model_t.
Definition: ngram_model_internal.h:55
sorted_list_t::free
int32 free
first free element in list
Definition: lm3g_model.h:84
ngram_word
SPHINXBASE_EXPORT const char * ngram_word(ngram_model_t *model, int32 wid)
Look up word string for numerical word ID.
Definition: ngram_model.c:782
fclose_comp
SPHINXBASE_EXPORT void fclose_comp(FILE *fp, int32 ispipe)
Close a file opened using fopen_comp.
Definition: pio.c:175
mmio_file_read
SPHINXBASE_EXPORT mmio_file_t * mmio_file_read(const char *filename)
Memory-map a file for reading.
Definition: mmio.c:206
bigram_s::wid
uint32 wid
Index of unigram entry for this.
Definition: ngram_model_arpa.h:53
ngram_model_mgrams
SPHINXBASE_EXPORT ngram_iter_t * ngram_model_mgrams(ngram_model_t *model, int m)
Iterate over all M-grams.
Definition: ngram_model.c:689
lm3g_model_s::tginfo
tginfo_t ** tginfo
tginfo[lw2] is head of linked list of trigram information for some cached subset of bigrams (*...
Definition: lm3g_model.h:154
lm3g_model_s::n_prob2
int32 n_prob2
prob2 size
Definition: lm3g_model.h:147
pio.h
file IO related operations.
logmath_log10_to_log
SPHINXBASE_EXPORT int logmath_log10_to_log(logmath_t *lmath, float64 log_p)
Convert base 10 log (in floating point) to integer log in base B.
Definition: logmath.c:474
lm3g_model_s::tseg_base
int32 * tseg_base
tseg_base[i>>LOG_BG_SEG_SZ] = index of 1st trigram for bigram segment (i>>LOG_BG_SEG_SZ) ...
Definition: lm3g_model.h:152
trigram_s::wid
uint32 wid
Index of unigram entry for this.
Definition: ngram_model_arpa.h:67