Source code for deeppavlov.models.spelling_correction.electors.kenlm_elector

# Copyright 2017 Neural Networks and Deep Learning lab, MIPT
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# See the License for the specific language governing permissions and
# limitations under the License.

from logging import getLogger
from pathlib import Path
from typing import List, Tuple

import kenlm

from deeppavlov.core.commands.utils import expand_path
from deeppavlov.core.common.registry import register
from deeppavlov.core.models.component import Component

logger = getLogger(__name__)

[docs]@register('kenlm_elector') class KenlmElector(Component): """Component that chooses a candidate with the highest product of base and language model probabilities Args: load_path: path to the kenlm model file beam_size: beam size for highest probability search Attributes: lm: kenlm object beam_size: beam size for highest probability search """ def __init__(self, load_path: Path, beam_size: int = 4, *args, **kwargs): self.lm = kenlm.Model(str(expand_path(load_path))) self.beam_size = beam_size
[docs] def __call__(self, batch: List[List[List[Tuple[float, str]]]]) -> List[List[str]]: """Choose the best candidate for every token Args: batch: batch of probabilities and string values of candidates for every token in a sentence Returns: batch of corrected tokenized sentences """ return [self._infer_instance(candidates) for candidates in batch]
def _infer_instance(self, candidates: List[List[Tuple[float, str]]]): candidates = candidates + [[(0, '</s>')]] state = kenlm.State() self.lm.BeginSentenceWrite(state) beam = [(0, state, [])] for sublist in candidates: new_beam = [] for beam_score, beam_state, beam_words in beam: for score, candidate in sublist: prev_state = beam_state c_score = 0 cs = candidate.split() for candidate in cs: state = kenlm.State() c_score += self.lm.BaseScore(prev_state, candidate, state) prev_state = state new_beam.append((beam_score + score + c_score, state, beam_words + cs)) new_beam.sort(reverse=True) beam = new_beam[:self.beam_size] score, state, words = beam[0] return words[:-1]