path: root/localedata/unicode-gen/ctype_compatibility.py

#!/usr/bin/python3
# -*- coding: utf-8 -*-
# Copyright (C) 2014-2016 Free Software Foundation, Inc.
# This file is part of the GNU C Library.
#
# The GNU C Library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# The GNU C Library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with the GNU C Library; if not, see
# <http://www.gnu.org/licenses/>.

'''
This script is useful for checking the differences between
an old LC_CTYPE file /usr/share/i18n/locale/i18n and a
new one generated by gen_unicode_ctype.py

To see how it is used, call it with the “-h” option:

    $ ./ctype_compatibility.py -h
    … prints usage message …
'''

import sys
import re
import unicodedata
import argparse

from ctype_compatibility_test_cases import TEST_CASES

def get_lines_from_file(filename):
    '''Get all non-comment lines from a i18n file

    Also merge all lines which are continued on the next line because
    they end in “/” into a single line.
    '''
    with open(filename) as i18n_file:
        current_line = ''
        for line in i18n_file:
            line = line.strip('\n')
            if '%' in line:
                if line.endswith('/'):
                    line = line[0:line.find('%')] + '/'
                else:
                    line = line[0:line.find('%')]
            line = line.strip()
            if line.endswith('/'):
                current_line += line[:-1]
            else:
                yield current_line + line
                current_line = ''
    if current_line: # file ends with a continuation line
        yield current_line

def extract_character_classes(filename):
    '''Get all Unicode code points for each character class from a file

    Store these code points in a dictionary using the character classes
    as keys and the list of code points in this character class as values.

    In case  of the character classes “toupper”, “tolower”, and “totitle”,
    these area actually pairs of code points
    '''
    ctype_dict = {}
    for line in get_lines_from_file(filename):
        for char_class in [
                'upper',
                'lower',
                'alpha',
                'digit',
                'outdigit',
                'space',
                'cntrl',
                'punct',
                'graph',
                'print',
                'xdigit',
                'blank',
                'combining',
                'combining_level3',
                'toupper',
                'tolower',
                'totitle']:
            match = re.match(r'^('
                             +'(?:(?:class|map)\s+")'
                             +re.escape(char_class)+
                             '(?:";)\s+'
                             +'|'
                             +re.escape(char_class)+'\s+'
                             +')', line)
            if match:
                if char_class not in ctype_dict:
                    ctype_dict[char_class] = []
                process_chars(
                    ctype_dict[char_class],
                    line[match.end():])
    return ctype_dict

def process_chars(char_class_list, code_point_line):
    '''
    Extract Unicode values from code_point_line
    and add to the list of code points in a character class
    '''
    for code_points in code_point_line.split(';'):
        code_points = code_points.strip()
        match = re.match(r'^<U(?P<codepoint>[0-9A-F]{4,8})>$', code_points)
        if match: # <Uxxxx>
            char_class_list.append(
                int(match.group('codepoint'), 16))
            continue
        match = re.match(
            r'^<U(?P<codepoint1>[0-9A-F]{4,8})>'
            +'\.\.'+
            '<U(?P<codepoint2>[0-9A-F]{4,8})>$',
            code_points)
        if match: # <Uxxxx>..<Uxxxx>
            for codepoint in range(
                    int(match.group('codepoint1'), 16),
                    int(match.group('codepoint2'), 16) + 1):
                char_class_list.append(codepoint)
            continue
        match = re.match(
            r'^<U(?P<codepoint1>[0-9A-F]{4,8})>'
            +'\.\.\(2\)\.\.'+
            '<U(?P<codepoint2>[0-9A-F]{4,8})>$',
            code_points)
        if match: # <Uxxxx>..(2)..<Uxxxx>
            for codepoint in range(
                    int(match.group('codepoint1'), 16),
                    int(match.group('codepoint2'), 16) + 1,
                    2):
                char_class_list.append(codepoint)
            continue
        match = re.match(
            r'^\('
            +'<U(?P<codepoint1>[0-9A-F]{4,8})>'
            +','+
            '<U(?P<codepoint2>[0-9A-F]{4,8})>'
            +'\)$',
            code_points)
        if match: # (<Uxxxx>,<Uxxxx>)
            char_class_list.append((
                int(match.group('codepoint1'), 16),
                int(match.group('codepoint2'), 16)))
            continue
        sys.stderr.write(
            ('None of the regexps matched '
             + 'code_points=%(cp)s in code_point_line=%(cpl)s\n') %{
            'cp': code_points,
            'cpl': code_point_line
        })
        exit(1)

def compare_lists(old_ctype_dict, new_ctype_dict):
    '''Compare character classes in the old and the new LC_CTYPE'''
    print('****************************************************')
    print('Character classes which are only in the new '
          + 'or only in the old file:')
    for char_class in sorted(old_ctype_dict):
        if char_class not in new_ctype_dict:
            print('Character class %s is in old ctype but not in new ctype'
                  %char_class)
    for char_class in sorted(new_ctype_dict):
        if char_class not in old_ctype_dict:
            print('Character class %s is in new ctype but not in old ctype'
                  %char_class)
    for char_class in sorted(old_ctype_dict):
        print("****************************************************")
        print("%s: %d chars in old ctype and %d chars in new ctype" %(
            char_class,
            len(old_ctype_dict[char_class]),
            len(new_ctype_dict[char_class])))
        print("----------------------------------------------------")
        report(char_class,
               old_ctype_dict[char_class],
               new_ctype_dict[char_class])

def report_code_points(char_class, code_point_list, text=''):
    '''Report all code points which have been added to or removed from a
    character class.
    '''
    for code_point in sorted(code_point_list):
        if type(code_point) == type(int()):
            print('%(char_class)s: %(text)s: %(char)s %(code_point)s %(name)s'
                  %{'text': text,
                    'char': chr(code_point),
                    'char_class': char_class,
                    'code_point': hex(code_point),
                    'name': unicodedata.name(chr(code_point), 'name unknown')})
        else:
            print(('%(char_class)s: %(text)s: '
                   + '%(char0)s → %(char1)s '
                   + '%(code_point0)s → %(code_point1)s '
                   + '%(name0)s → %(name1)s') %{
                'text': text,
                'char_class': char_class,
                'char0': chr(code_point[0]),
                'code_point0': hex(code_point[0]),
                'name0': unicodedata.name(chr(code_point[0]), 'name unknown'),
                'char1': chr(code_point[1]),
                'code_point1': hex(code_point[1]),
                'name1': unicodedata.name(chr(code_point[1]), 'name unknown')
            })

def report(char_class, old_list, new_list):
    '''Report the differences for a certain LC_CTYPE character class
    between the old and the newly generated state
    '''
    missing_chars = list(set(old_list)-set(new_list))
    print(('%(char_class)s: Missing %(number)d characters '
           + 'of old ctype in new ctype ')
          %{'char_class': char_class, 'number': len(missing_chars)})
    if ARGS.show_missing_characters:
        report_code_points(char_class, missing_chars, 'Missing')
    added_chars = list(set(new_list)-set(old_list))
    print(('%(char_class)s: Added %(number)d characters '
           + 'in new ctype which were not in old ctype')
          %{'char_class': char_class, 'number': len(added_chars)})
    if ARGS.show_added_characters:
        report_code_points(char_class, added_chars, 'Added')


def cperror(error_message, errorcounter=0):
    '''Increase number of errors by one and print an error message'''
    print(error_message)
    return errorcounter + 1

def cpcheck(ctype_dict, code_point_list_with_ranges, char_classes, reason='',
            errorcounter=0):
    '''The parameter “code_point_list_with_ranges” is a list of
    integers or pairs of integers, for example:

    [0x0E31, (0x0E34, 0x0E3A), (0x0E47, 0x0E4E)]

    where the pairs of integers stand for all the code points in the range
    of the two integers given, including the two integers of the pair.

    '''
    for code_point_range in code_point_list_with_ranges:
        for code_point in ([code_point_range]
                           if type(code_point_range) == type(int())
                           else range(code_point_range[0],
                                      code_point_range[1]+1)):
            for char_class_tuple in char_classes:
                char_class = char_class_tuple[0]
                in_char_class = char_class_tuple[1]
                if (code_point in ctype_dict[char_class]) != in_char_class:
                    errorcounter = cperror(
                        ('error: %(code_point)s %(char)s '
                         + '%(char_class)s %(in)s: %(reason)s') %{
                             'code_point': hex(code_point),
                             'char': chr(code_point),
                             'char_class': char_class,
                             'in': not in_char_class,
                             'reason': reason},
                        errorcounter)
    return errorcounter

def tests(ctype_dict, errorcounter = 0):
    '''Test a LC_CTYPE character class dictionary for known errors'''
    # copy the information from ctype_dict (which contains lists) in
    # a new dictionary ctype_dict2 (which contains dictionaries).
    # The checks below are easier with that type of data structure.

    ctype_dict2 = {}
    for key in ctype_dict:
        ctype_dict2[key] = {}
        if ctype_dict[key]:
            if type(ctype_dict[key][0]) == type(int()):
                for value in ctype_dict[key]:
                    ctype_dict2[key][value] = 1
            else: # key is 'toupper', 'tolower', or 'totitle'
                for value in ctype_dict[key]:
                    ctype_dict2[key][value[0]] = value[1]

    for test_case in TEST_CASES:
        errorcounter = cpcheck(ctype_dict2,
                               test_case[0],
                               test_case[1],
                               test_case[2],
                               errorcounter = errorcounter)

    for code_point in range(0, 0x110000):
        # toupper restriction: "Only characters specified for the keywords
	# lower and upper shall be specified.
        if (code_point in ctype_dict2['toupper']
            and code_point != ctype_dict2['toupper'][code_point]
            and not (code_point in ctype_dict2['lower']
                     or code_point in ctype_dict2['upper'])):
            errorcounter = cperror(
                ('error: %(char1)s is not upper|lower '
                 + 'but toupper(%(cp1)s)=%(cp2)s (%(char2)s)') %{
                     'char1': chr(code_point),
                     'cp1': hex(code_point),
                     'cp2': hex(ctype_dict2['toupper'][code_point]),
                     'char2': chr(ctype_dict2['toupper'][code_point])
                 },
                errorcounter)
        # tolower restriction: "Only characters specified for the keywords
	# lower and upper shall be specified.
        if (code_point in ctype_dict2['tolower']
            and code_point != ctype_dict2['tolower'][code_point]
            and not (code_point in ctype_dict2['lower']
                     or code_point in ctype_dict2['upper'])):
            errorcounter = cperror(
                ('error: %(char1)s is not upper|lower '
                 + 'but tolower(%(cp1)s)=%(cp2)s (%(char2)s)') %{
                     'char1': chr(code_point),
                     'cp1': hex(code_point),
                     'cp2': hex(ctype_dict2['tolower'][code_point]),
                     'char2': chr(ctype_dict2['tolower'][code_point])
                 },
                errorcounter)
        # alpha restriction: "Characters classified as either upper or lower
	# shall automatically belong to this class.
        if ((code_point in ctype_dict2['lower']
             or code_point in ctype_dict2['upper'])
            and code_point not in ctype_dict2['alpha']):
            errorcounter = cperror(
                'error: %(char)s %(cp)s is upper|lower but not alpha' %{
                    'char': chr(code_point),
                    'cp': hex(code_point)
                },
                errorcounter)
        # alpha restriction: "No character specified for the keywords cntrl,
	# digit, punct or space shall be specified."
        if (code_point in ctype_dict2['alpha']
            and code_point in ctype_dict2['cntrl']):
            errorcounter = cperror(
                'error: %(char)s %(cp)s is alpha and cntrl' %{
                    'char': chr(code_point),
                    'cp': hex(code_point)
                },
                errorcounter)
        if (code_point in ctype_dict2['alpha']
            and code_point in ctype_dict2['digit']):
            errorcounter = cperror(
                'error: %(char)s %(cp)s is alpha and digit' %{
                    'char': chr(code_point),
                    'cp': hex(code_point)
                },
                errorcounter)
        if (code_point in ctype_dict2['alpha']
            and code_point in ctype_dict2['punct']):
            errorcounter = cperror(
                'error: %(char)s %(cp)s is alpha and punct' %{
                    'char': chr(code_point),
                    'cp': hex(code_point)
                },
                errorcounter)
        if (code_point in ctype_dict2['alpha']
            and code_point in ctype_dict2['space']):
            errorcounter = cperror(
                'error: %(char)s %(cp)s is alpha and space' %{
                    'char': chr(code_point),
                    'cp': hex(code_point)
                },
                errorcounter)
        # space restriction: "No character specified for the keywords upper,
	# lower, alpha, digit, graph or xdigit shall be specified."
	# upper, lower, alpha already checked above.
        if (code_point in ctype_dict2['space']
            and code_point in ctype_dict2['digit']):
            errorcounter = cperror(
                'error: %(char)s %(cp)s is space and digit' %{
                    'char': chr(code_point),
                    'cp': hex(code_point)
                },
                errorcounter)
        if (code_point in ctype_dict2['space']
            and code_point in ctype_dict2['graph']):
            errorcounter = cperror(
                'error: %(char)s %(cp)s is space and graph' %{
                    'char': chr(code_point),
                    'cp': hex(code_point)
                },
                errorcounter)
        if (code_point in ctype_dict2['space']
            and code_point in ctype_dict2['xdigit']):
            errorcounter = cperror(
                'error: %(char)s %(cp)s is space and xdigit' %{
                    'char': chr(code_point),
                    'cp': hex(code_point)
                },
                errorcounter)
        # cntrl restriction: "No character specified for the keywords upper,
	# lower, alpha, digit, punct, graph, print or xdigit shall be
	# specified."  upper, lower, alpha already checked above.
        if (code_point in ctype_dict2['cntrl']
            and code_point in ctype_dict2['digit']):
            errorcounter = cperror(
                'error: %(char)s %(cp)s is cntrl and digit' %{
                    'char': chr(code_point),
                    'cp': hex(code_point)
                },
                errorcounter)
        if (code_point in ctype_dict2['cntrl']
            and code_point in ctype_dict2['punct']):
            errorcounter = cperror(
                'error: %(char)s %(cp)s is cntrl and punct' %{
                    'char': chr(code_point),
                    'cp': hex(code_point)
                },
                errorcounter)
        if (code_point in ctype_dict2['cntrl']
            and code_point in ctype_dict2['graph']):
            errorcounter = cperror(
                'error: %(char)s %(cp)s is cntrl and graph' %{
                    'char': chr(code_point),
                    'cp': hex(code_point)
                },
                errorcounter)
        if (code_point in ctype_dict2['cntrl']
            and code_point in ctype_dict2['print']):
            errorcounter = cperror(
                'error: %(char)s %(cp)s is cntrl and print' %{
                    'char': chr(code_point),
                    'cp': hex(code_point)
                },
                errorcounter)
        if (code_point in ctype_dict2['cntrl']
            and code_point in ctype_dict2['xdigit']):
            errorcounter = cperror(
                'error: %(char)s %(cp)s is cntrl and xdigit' %{
                    'char': chr(code_point),
                    'cp': hex(code_point)
                },
                errorcounter)
        # punct restriction: "No character specified for the keywords upper,
	# lower, alpha, digit, cntrl, xdigit or as the <space> character shall
	# be specified."  upper, lower, alpha, cntrl already checked above.
        if (code_point in ctype_dict2['punct']
            and code_point in ctype_dict2['digit']):
            errorcounter = cperror(
                'error: %(char)s %(cp)s is punct and digit' %{
                    'char': chr(code_point),
                    'cp': hex(code_point)
                },
                errorcounter)
        if (code_point in ctype_dict2['punct']
            and code_point in ctype_dict2['xdigit']):
            errorcounter = cperror(
                'error: %(char)s %(cp)s is punct and xdigit' %{
                    'char': chr(code_point),
                    'cp': hex(code_point)
                },
                errorcounter)
        if (code_point in ctype_dict2['punct']
            and code_point == 0x0020):
            errorcounter = cperror(
                'error: %(char)s %(cp)s is punct.' %{
                    'char': chr(code_point),
                    'cp': hex(code_point)
                },
                errorcounter)
        # graph restriction: "No character specified for the keyword cntrl
	# shall be specified."  Already checked above.

        # print restriction: "No character specified for the keyword cntrl
	# shall be specified."  Already checked above.

        # graph - print relation: differ only in the <space> character.
	# How is this possible if there are more than one space character?!
	# I think susv2/xbd/locale.html should speak of "space characters",
	# not "space character".
        if (code_point in ctype_dict2['print']
            and not (code_point in ctype_dict2['graph']
                     or code_point in ctype_dict2['space'])):
            errorcounter = cperror(
                'error: %(char)s %(cp)s is print but not graph|space' %{
                    'char': chr(code_point),
                    'cp': hex(code_point)
                },
                errorcounter)
        if (code_point not in ctype_dict2['print']
            and (code_point in ctype_dict2['graph']
                 or code_point ==  0x0020)):
            errorcounter = cperror(
                'error: %(char)s %(cp)s graph|space but not print' %{
                    'char': chr(code_point),
                    'cp': hex(code_point)
                },
                errorcounter)
    return errorcounter

if __name__ == "__main__":
    PARSER = argparse.ArgumentParser(
        description='''
        Compare the contents of LC_CTYPE in two files and check for errors.
        ''')
    PARSER.add_argument(
        '-o', '--old_ctype_file',
        nargs='?',
        type=str,
        default='i18n',
        help='The old ctype file, default: %(default)s')
    PARSER.add_argument(
        '-n', '--new_ctype_file',
        nargs='?',
        type=str,
        default='unicode-ctype',
        help='The new ctype file, default: %(default)s')
    PARSER.add_argument(
        '-a', '--show_added_characters',
        action='store_true',
        help=('Show characters which were added to each '
              + 'character class in detail.'))
    PARSER.add_argument(
        '-m', '--show_missing_characters',
        action='store_true',
        help=('Show characters which were removed from each '
              + 'character class in detail.'))
    ARGS = PARSER.parse_args()

    OLD_CTYPE_DICT = extract_character_classes(
        ARGS.old_ctype_file)
    NEW_CTYPE_DICT = extract_character_classes(
        ARGS.new_ctype_file)
    compare_lists(OLD_CTYPE_DICT, NEW_CTYPE_DICT)
    print('============================================================')
    print('Checking for errors in old ctype file: %s' %ARGS.old_ctype_file)
    print('------------------------------------------------------------')
    NUMBER_OF_ERRORS_IN_OLD_FILE = tests(OLD_CTYPE_DICT, errorcounter = 0)
    print('------------------------------------------------------------')
    print('Old file = %s' %ARGS.old_ctype_file)
    print('Number of errors in old file = %s' %NUMBER_OF_ERRORS_IN_OLD_FILE)
    print('------------------------------------------------------------')
    print('============================================================')
    print('Checking for errors in new ctype file: %s' %ARGS.new_ctype_file)
    print('------------------------------------------------------------')
    NUMBER_OF_ERRORS_IN_NEW_FILE = tests(NEW_CTYPE_DICT, errorcounter = 0)
    print('------------------------------------------------------------')
    print('New file = %s' %ARGS.new_ctype_file)
    print('Number of errors in new file = %s' %NUMBER_OF_ERRORS_IN_NEW_FILE)
    print('------------------------------------------------------------')
    if NUMBER_OF_ERRORS_IN_NEW_FILE > 0:
        exit(1)
    else:
        exit(0)