OpenCVやってみる-38. 処理の調整2

春のパン祭り点数文字認識処理を調整していますが、なかなか完璧にはいかず…

調整は今回までにして、実際のアプリケーションを作っていく方向に進めたいと思います。

今回変更したこと

色々調整していて、以下変更しました。

• 数字テンプレートとする輪郭データを、cv2.minAreaRect()での角度を元にまっすぐに直しておく。(効果はあまり出なかったが、処理を整理できたので、この変更版処理を使っておきたい)
• ICP処理の中で、最近傍点探索処理があったが、この高速化を行った。
• 点数文字認識処理の区切り方を修正した。
• 一致度計算では、cv2.matchTemplate()関数を使っていたが、2つのサイズの一致した2値画像を比較したい、というだけなので、それに合わせた処理に変更。パフォーマンスは確認していないが、速くなっているんではないかと。

今まで作成した処理

色々と変更しながらやってきたので、現状のバージョンを以下並べていきます。

ライブラリインポート、画像データ読み込み

import cv2
import numpy as np
%matplotlib inline
from matplotlib import pyplot as plt
import math
import copy
import random

img1 = cv2.imread('harupan_190428_1.jpg')
img2 = cv2.imread('harupan_190428_2.jpg')
img3 = cv2.imread('harupan_200317_1.jpg')
img4 = cv2.imread('harupan_210227_2.jpg')
img5 = cv2.imread('harupan_210402_1.jpg')
img6 = cv2.imread('harupan_210402_2.jpg')
img7 = cv2.imread('harupan_210414_1.jpg')

点数文字候補輪郭の検出

def detect_candidate_contours(image, res_th=800):
    h, w, chs = image.shape
    if h > res_th or w > res_th:
        k = float(res_th)/h if w > h else float(res_th)/w
    else:
        k = 1.0
    img = cv2.resize(image, None, fx=k, fy=k, interpolation=cv2.INTER_AREA)
    hsv = cv2.cvtColor(img, cv2.COLOR_BGR2HSV)
    # Convert hue value (rotation, mask by saturation)
    hsv[:,:,0] = np.where(hsv[:,:,0] < 50, hsv[:,:,0]+180, hsv[:,:,0])
    hsv[:,:,0] = np.where(hsv[:,:,1] < 100, 0, hsv[:,:,0])
    # Thresholding with cv2.inRange()
    th_hue = cv2.inRange(hsv[:,:,0], 135, 190)
    # Retrieve all points on the contours (cv2.CHAIN_APPROX_NONE)
    contours, hierarchy = cv2.findContours(th_hue, cv2.RETR_TREE, cv2.CHAIN_APPROX_NONE)
    indices0 = [i for i,hier in enumerate(hierarchy[0,:,:]) if hier[3] == -1]
    indices1 = [i for i,hier in enumerate(hierarchy[0,:,:]) if hier[3] in indices0]
    contours1 = [contours[i] for i in indices1]
    contours1_filtered = [ctr for ctr in contours1 if cv2.contourArea(ctr) > float(res_th)*float(res_th)/4000]
    return contours1_filtered, img

補助処理

• 輪郭周辺の小画像作成
  輪郭周辺の画像領域の切り出し、画像と輪郭自体の原点もその領域の原点に直す。
  比較対象データの作成が一つの目的、もう一つの目的はデバッグ。
• 輪郭の塗りつぶし画像作成
  テンプレートデータ作成用、比較用データ作成用

def create_contour_area_image(img, ctr):
    x,y,w,h = cv2.boundingRect(ctr)
    rtn_img = img[y:y+h,x:x+w,:].copy()
    rtn_ctr = ctr.copy()
    origin = np.array([x,y])
    for c in rtn_ctr:
        c[0,:] -= origin
    return rtn_img, rtn_ctr

# ctr: Should be output of create_contour_area_image() (Origin of points is the origin of bounding box)
# img_shape: Optional, tuple of (image_height, image_width), if omitted, calculated from ctr
def create_solid_contour(ctr, img_shape=(int(0),int(0))):
    if img_shape == (int(0),int(0)):
        _,_,w,h = cv2.boundingRect(ctr)
    else:
        h,w = img_shape
    img = np.zeros((h,w), 'uint8')
    img = cv2.drawContours(img, [ctr], -1, 255, -1)
    return img

# ctr: Should be output of create_contour_area_image() (Origin of points is the origin of bounding box)
def create_upright_solid_contour(ctr):
    (cx,cy),(w,h),angle = cv2.minAreaRect(ctr)
    M = cv2.getRotationMatrix2D((cx,cy), angle, 1)
    for i in range(ctr.shape[0]):
        ctr[i,0,:] = ( M @ np.array([ctr[i,0,0], ctr[i,0,1], 1]) ).astype('int')
    rect = cv2.boundingRect(ctr)
    img = np.zeros((rect[3],rect[2]), 'uint8')
    ctr -= rect[0:2]
    M[:,2] -= rect[0:2]
    img = cv2.drawContours(img, [ctr], -1, 255,-1)
    return img, M, ctr

点数文字認識処理の整理

データセット

各輪郭について、処理の中で何度か使われる生成データ。
クラスでまとめておく。

class contour_dataset:
    def __init__(self, ctr):
        self.ctr = ctr.copy()
        self.rrect = cv2.minAreaRect(ctr)
        self.box = cv2.boxPoints(self.rrect)
        self.solid = create_solid_contour(ctr)
        self.pts = np.array([p for p in ctr[:,0,:]])

class template_dataset:
    def __init__(self, ctr, num, selected_idx=[0]):
        self.ctr = ctr.copy()
        self.num = num
        self.rrect = cv2.minAreaRect(ctr)
        self.box = cv2.boxPoints(self.rrect)
        if num == 0:
            self.solid,_,_ = create_upright_solid_contour(ctr)
        else:
            self.solid = create_solid_contour(ctr)
        self.pts = np.array([ctr[idx,0,:] for idx in selected_idx])

ICP処理

補助処理と本体。

• 最近傍点探索処理は高速化版。
  2点間の距離を計算していましたが、そうするとルート計算が入って計算が重くなるかと。
  距離の2乗の計算であれば計算が軽く、また、大小比較をするだけなら2乗したものでも問題ないので、そのように変更しました。

# pts: list of 2D points, or ndarray of shape (n,2)
# query: 2D point to find nearest neighbor
def find_nearest_neighbor(pts, query):
    min_distance_sq = float('inf')
    min_idx = 0
    for i, p in enumerate(pts):
        d = np.dot(query - p, query - p)
        if(d < min_distance_sq):
            min_distance_sq = d
            min_idx = i
    return min_idx, np.sqrt(min_distance_sq)

# src, dst: ndarray, shape is (n,2) (n: number of points)
def estimate_affine_2d(src, dst):
    n = min(src.shape[0], dst.shape[0])
    x = dst[0:n].flatten()
    A = np.zeros((2*n,6))
    for i in range(n):
        A[i*2,0] = src[i,0]
        A[i*2,1] = src[i,1]
        A[i*2,2] = 1
        A[i*2+1,3] = src[i,0]
        A[i*2+1,4] = src[i,1]
        A[i*2+1,5] = 1
    M = np.linalg.inv(A.T @ A) @ A.T @ x
    return M.reshape([2,3])

# Find optimum affine matrix using ICP algorithm
# src_pts: ndarray, shape is (n_s,2) (n_s: number of points)
# dst_pts: ndarray, shape is (n_d,2) (n_d: number of points, n_d should be larger or equal to n_s)
# initial_matrix: ndarray, shape is (2,3)
def icp(src_pts, dst_pts, max_iter=20, initial_matrix=np.array([[1.0, 0.0, 0.0], [0.0, 1.0, 0.0]])):
    default_affine_matrix = np.array([[1.0, 0.0, 0.0], [0.0, 1.0, 0.0]])
    if dst_pts.shape[0] < src_pts.shape[0]:
        print("icp: Insufficient destination points")
        return default_affine_matrix, False
    if initial_matrix.shape != (2,3):
        print("icp: Illegal shape of initial_matrix")
        return default_affine_matrix, False
    M = initial_matrix
    # Store indices of the nearest neighbor point of dst_pts to the converted point of src_pts
    nn_idx = []
    for i in range(max_iter):
        nn_idx_tmp = []
        dst_pts_list = [p for p in dst_pts]
        idx_list = list(range(0,dst_pts.shape[0]))
        for p in src_pts:
            p2 = M @ np.array([p[0], p[1], 1])
            idx, d = find_nearest_neighbor(dst_pts_list, p2)
            nn_idx_tmp += [idx_list[idx]]
            del dst_pts_list[idx]
            del idx_list[idx]
        if nn_idx != [] and nn_idx == nn_idx_tmp:
            break
        dst_pts2 = np.zeros_like(src_pts)
        for j,idx in enumerate(nn_idx_tmp):
            dst_pts2[j,:] = dst_pts[idx,:]
        M = estimate_affine_2d(src_pts, dst_pts2)
        nn_idx = nn_idx_tmp
        if i == max_iter -1:
            return M, False
    return M, True

一致度計算処理

今まで使っていたものから少しバグ修正(デバッグ用に返す画像が間違っていた)があるぐらい。

def binary_image_similarity(img1, img2):
    if img1.shape != img2.shape:
        print('binary_image_similarity: Different image size')
        return 0.0
    xor_img = cv2.bitwise_xor(img1, img2)
    return 1.0 - np.float(np.count_nonzero(xor_img)) / (img1.shape[0]*img2.shape[1])

# src, dst: contour_dataset or template_dataset (holding member variables box, solid)
def get_transform_by_rotated_rectangle(src, dst):
    # Rotated patterns are created when starting index is slided
    dst_box2 = np.vstack([dst.box, dst.box])
    max_similarity = 0.0
    max_converted_img = np.zeros((dst.solid.shape[1], dst.solid.shape[0]), 'uint8')
    for i in range(4):
        M = cv2.getAffineTransform(src.box[0:3], dst_box2[i:i+3])
        converted_img = cv2.warpAffine(src.solid, M, dsize=(dst.solid.shape[1], dst.solid.shape[0]), flags=cv2.INTER_NEAREST)
        similarity = binary_image_similarity(converted_img, dst.solid)
        if similarity > max_similarity:
            M_rtn = M
            max_similarity = similarity
            max_converted_img = converted_img
    return M_rtn, max_similarity, max_converted_img

def get_similarity_with_template(target_data, template_data, sim_th_high=0.95, sim_th_low=0.7):
    _,(w1,h1), _ = target_data.rrect
    _,(w2,h2), _ = template_data.rrect
    r = w1/h1 if w1 < h1 else h1/w1
    r = r * h2/w2 if w2 < h2 else r * w2/h2
    M, sim_init, _ = get_transform_by_rotated_rectangle(template_data, target_data)
    if sim_init > sim_th_high or sim_init < sim_th_low or r > 1.4 or r < 0.7:
        dsize = (template_data.solid.shape[1], template_data.solid.shape[0])
        flags = cv2.INTER_NEAREST|cv2.WARP_INVERSE_MAP
        converted_img = cv2.warpAffine(target_data.solid, M, dsize=dsize, flags=flags)
        return sim_init, converted_img
    M, _ = icp(template_data.pts, target_data.pts, initial_matrix=M)
    Minv = cv2.invertAffineTransform(M)
    converted_ctr = np.zeros_like(target_data.ctr)
    for i in range(target_data.ctr.shape[0]):
        converted_ctr[i,0,:] = (Minv[:,0:2] @ target_data.ctr[i,0,:]) + Minv[:,2]
    converted_img = create_solid_contour(converted_ctr, img_shape=template_data.solid.shape)
    val = binary_image_similarity(converted_img, template_data.solid)
    return val, converted_img

def get_similarity_with_template_zero(target_data, template_data):
    dsize = (template_data.solid.shape[1], template_data.solid.shape[0])
    converted_img = cv2.resize(target_data.solid, dsize=dsize, interpolation=cv2.INTER_NEAREST)
    val = binary_image_similarity(converted_img, template_data.solid)
    return val, converted_img

def get_similarities(target, templates):
    similarities = []
    converted_imgs = []
    for tmpl in templates:
        if tmpl.num == 0:
            sim,converted_img = get_similarity_with_template_zero(target, tmpl)
        else:
            sim,converted_img = get_similarity_with_template(target, tmpl)
        similarities += [sim]
        converted_imgs += [converted_img]
    return similarities, converted_imgs

# target: Single contour to compare
# templates: List of template_dataset (for numbers 0, 1, 2, 3, 5)
# svm: Trained SVM
# return: determined number (0,1,2,3,5), -1 if none corresponds
def determine_number(target, templates, svm):
    similarities,_ = get_similarities(target, templates)
    _, result = svm.predict(np.array(similarities))
    return int(result[0])

SVM学習関連処理

学習用データのサンプルでは、同じ状況の再現のため、乱数のシードを指定できるようにしました。

def get_random_sample(data_in, labels_in, selected_labels, n_samples, seed=None):
    random.seed(seed)
    data_rtn = []
    labels_rtn = []
    for lab in selected_labels:
        samples = [d for i,d in enumerate(data_in) if labels_in[i]==lab]
        n = min(n_samples, len(samples))
        data_rtn += random.sample(samples, n)
        labels_rtn += [lab] * n
    return data_rtn, labels_rtn

def prepare_svm(train_data, train_labels):
    svm = cv2.ml.SVM_create()
    svm.setKernel(cv2.ml.SVM_LINEAR)
    svm.setType(cv2.ml.SVM_C_SVC)
    svm.setC(100)
    svm.setGamma(1)
    svm.train(np.array(train_data, 'float32'), cv2.ml.ROW_SAMPLE, np.array(train_labels))
    return svm

def print_stat(svm_results, svm_labels):
    stats = {k:{k2:0 for k2 in [-1, 0, 1, 2, 3, 5]} for k in [-1, 0, 1, 2, 3, 5]}
    for res, lab in zip(svm_results[1], svm_labels):
        stats[lab][int(res[0])] += 1
    for k,v in stats.items():
        print('label {:>2}'.format(k), ': {', end='')
        for k2,v2 in v.items():
            print('{}: {:>2}, '.format(k2,v2), end='')
        print('}')

def print_similarity_vector(sim, end=''):
    print('[',end='')
    for s in sim: print('{:.3f}, '.format(s), end='')
    print(']', end=end)

データの用意

ここから、実データからの必要なデータ取り出しを行っていきます。
一部のデータは、後で再利用のために保存します。

輪郭データの取得

original_imgs = [img1, img2, img3, img4, img5, img6, img7]
resized_imgs = []
resized_ctrs = []
original_img_idx = []
subctrs_all = []
subimgs_all = []
for idx, original_img in enumerate(original_imgs):
    ctrs, img = detect_candidate_contours(original_img)
    resized_imgs += [img]
    resized_ctrs += [ctrs]
    
    for ctr in ctrs:
        original_img_idx += [idx]
        subimg,subctr = create_contour_area_image(img, ctr)
        subctrs_all += [subctr]
        subimgs_all += [subimg]

テンプレートデータの用意

今までやっていた通り、3つ目の画像(2020年)、5つ目の画像(2021年)では、"3点"のデータがないので、1つ目の画像(2019年)のデータで代用します。

ctrs1_idx_zero = 26
ctrs1_idx_one = 27
ctrs1_idx_two = 24
ctrs1_idx_three = 33
ctrs1_idx_five = 8
ctrs1_idx_numbers = [ctrs1_idx_zero, ctrs1_idx_one, ctrs1_idx_two, ctrs1_idx_three, ctrs1_idx_five]

subimgs1 = []
subctrs1 = []
binimgs1 = []
subctrs1_selected_pts = []
for i,idx in enumerate(ctrs1_idx_numbers):
    img, ctr = create_contour_area_image(resized_imgs[0], resized_ctrs[0][idx])
    binimg, M, ctr2 = create_upright_solid_contour(ctr)
    img2 = cv2.warpAffine(img.copy(), M, (binimg.shape[1], binimg.shape[0]))
    subimgs1 += [img2]
    subctrs1 += [ctr2]
    binimgs1 += [binimg]
    ctr_selected_pts = [j for j in range(ctr2.shape[0]) if j % 5 == 0]
    if i != 0:
        subctrs1_selected_pts += [ctr_selected_pts]
    ctr_img = cv2.drawContours(img2.copy(), [ctr2], -1, (0,255,0), 2)
    pts_img = img2.copy()
    for p in ctr_selected_pts:
        pts_img = cv2.drawMarker(pts_img, ctr2[p,0,:], (0,255,0), markerType=cv2.MARKER_CROSS, markerSize=3)
    plt.subplot(3,5,1+i), plt.imshow(cv2.cvtColor(ctr_img, cv2.COLOR_BGR2RGB)), plt.xticks([]), plt.yticks([])
    plt.subplot(3,5,6+i), plt.imshow(binimg,cmap='gray'), plt.xticks([]), plt.yticks([])
    plt.subplot(3,5,11+i), plt.imshow(cv2.cvtColor(pts_img, cv2.COLOR_BGR2RGB), cmap='gray'), plt.xticks([]), plt.yticks([])
plt.show()

ctrs3_idx_zero = 7
ctrs3_idx_one = 4
ctrs3_idx_two = 17
ctrs3_idx_five = 6
ctrs3_idx_numbers = [ctrs3_idx_zero, ctrs3_idx_one, ctrs3_idx_two, ctrs3_idx_five]

subimgs3 = []
subctrs3 = []
binimgs3 = []
subctrs3_selected_pts = []
for i,idx in enumerate(ctrs3_idx_numbers):
    img, ctr = create_contour_area_image(resized_imgs[2], resized_ctrs[2][idx])
    binimg, M, ctr2 = create_upright_solid_contour(ctr)
    img2 = cv2.warpAffine(img.copy(), M, (binimg.shape[1], binimg.shape[0]))
    subimgs3 += [img2]
    subctrs3 += [ctr2]
    binimgs3 += [binimg]
    ctr_selected_pts = [j for j in range(ctr2.shape[0]) if j% 5 == 0]
    if i != 0:
        subctrs3_selected_pts += [ctr_selected_pts]
    ctr_img = cv2.drawContours(img2.copy(), [ctr2], -1, (0,255,0), 2)
    pts_img = img2.copy()
    for p in ctr_selected_pts:
        pts_img = cv2.drawMarker(pts_img, ctr2[p,0,:], (0,255,0), markerType=cv2.MARKER_CROSS, markerSize=3)
    plt.subplot(3,4,1+i), plt.imshow(cv2.cvtColor(ctr_img, cv2.COLOR_BGR2RGB)), plt.xticks([]), plt.yticks([])
    plt.subplot(3,4,5+i), plt.imshow(binimg,cmap='gray'), plt.xticks([]), plt.yticks([])
    plt.subplot(3,4,9+i), plt.imshow(cv2.cvtColor(pts_img, cv2.COLOR_BGR2RGB)), plt.xticks([]), plt.yticks([])
plt.show()

subimgs3.insert(3, subimgs1[3])
subctrs3.insert(3, subctrs1[3])
binimgs3.insert(3, binimgs1[3])
subctrs3_selected_pts.insert(2, subctrs1_selected_pts[2])

ctrs5_idx_zero = 3
ctrs5_idx_one = 4
ctrs5_idx_two = 2
ctrs5_idx_five = 5
ctrs5_idx_numbers = [ctrs5_idx_zero, ctrs5_idx_one, ctrs5_idx_two, ctrs5_idx_five]

subimgs5 = []
subctrs5 = []
binimgs5 = []
subctrs5_selected_pts = []
for i,idx in enumerate(ctrs5_idx_numbers):
    img, ctr = create_contour_area_image(resized_imgs[4], resized_ctrs[4][idx])
    binimg, M, ctr2 = create_upright_solid_contour(ctr)
    img2 = cv2.warpAffine(img.copy(), M, (binimg.shape[1], binimg.shape[0]))
    subimgs5 += [img2]
    subctrs5 += [ctr2]
    binimgs5 += [binimg]
    ctr_selected_pts = [j for j in range(ctr2.shape[0]) if j % 5 == 0]
    if i != 0:
        subctrs5_selected_pts += [ctr_selected_pts]
    ctr_img = cv2.drawContours(img2.copy(), [ctr2], -1, (0,255,0), 2)
    pts_img = img2.copy()
    for p in ctr_selected_pts:
        pts_img = cv2.drawMarker(pts_img, ctr2[p,0,:], (0,255,0), markerType=cv2.MARKER_CROSS, markerSize=3)
    plt.subplot(3,4,1+i), plt.imshow(cv2.cvtColor(ctr_img, cv2.COLOR_BGR2RGB)), plt.xticks([]), plt.yticks([])
    plt.subplot(3,4,5+i), plt.imshow(binimg,cmap='gray'), plt.xticks([]), plt.yticks([])
    plt.subplot(3,4,9+i), plt.imshow(cv2.cvtColor(pts_img, cv2.COLOR_BGR2RGB)), plt.xticks([]), plt.yticks([])
plt.show()

subimgs5.insert(3, subimgs1[3])
subctrs5.insert(3, subctrs1[3])
binimgs5.insert(3, binimgs1[3])
subctrs5_selected_pts.insert(2, subctrs1_selected_pts[2])

正解ラベル

labels1 = [-1,-1,-1,-1,-1
           ,-1,5,0,5,1
           ,5,0,2,1,2
           ,-1,-1,1,1,5
           ,0,2,5,0,2
           ,5,0,1,2,-1
           ,5,1,2,3,1
           ,5,0,-1]

labels2 = [-1,-1,-1,-1,-1
           ,-1,5,0,5,1
           ,5,0,2,1,2
           ,-1,-1,1,1,5
           ,0,-1,2,5,0
           ,2,5,0,1,2
           ,5,0,1,-1,2
           ,-1,-1,-1,3,-1
           ,5,0,-1,1,-1
           ,-1]

labels3 = [-1,-1,-1,-1,1
           ,1,5,0,1,1
           ,5,0,5,0,-1
           ,-1,-1,2,-1,-1
           ,-1,1,1,1,-1
           ,1,-1,-1,1,1
           ,-1,2,-1,1,-1
           ,1,2,-1,1,-1
           ,-1,2,5,-1,0
           ,-1,1,1]

labels4 = [-1,-1,-1,-1,-1
           ,-1,-1,-1,-1,-1
           ,-1,-1,-1,-1,-1
           ,-1,-1,-1,1,1
           ,1,1,1,1,1
           ,-1,5,0,2,5
           ,0,2,1,2,2
           ,-1,-1,-1,1,1
           ,1]

labels5 = [-1,-1,2,0,1
           ,5,-1,1,1,1
           ,1,1,1,1,1
           ,1,-1,5,1,0
           ,5,1,2,0,5
           ,0,2,1,2,2
           ,-1,-1,1,1,1
           ]

labels6 = [-1,0,1,5,2
            ,-1,1,1,1,1
            ,5,1,0,5,0
            ,2,1,5,0,2
            ,2,2,1,-1,-1
            ,1,1,1,1,1
            ,1,1,1]

labels7 = [-1,-1,-1,-1,-1
           ,-1,1,2,2,2
           ,2,1,2,2,2
           ,1,-1,-1,-1,2
           ,1,2,1,1]

labels_all = labels1 + labels2 + labels3 + labels4 + labels5 + labels6 + labels7

データセットの作成

# Prepare template data for "0"
templates1 = [template_dataset(subctrs1[0], 0)]
templates3 = [template_dataset(subctrs3[0], 0)]
templates5 = [template_dataset(subctrs5[0], 0)]
# Prepare template data for other numbers
numbers = [1, 2, 3, 5]
for i,num in enumerate(numbers):
    templates1 += [template_dataset(subctrs1[i+1], num, subctrs1_selected_pts[i])]
    templates3 += [template_dataset(subctrs3[i+1], num, subctrs3_selected_pts[i])]
    templates5 += [template_dataset(subctrs5[i+1], num, subctrs5_selected_pts[i])]
ctr_datasets_all = [contour_dataset(ctr) for ctr in subctrs_all]

一致度計算実施

templates_sel = [1,1,3,5,5,5,5]
def select_template(i):
    img_idx = original_img_idx[i]
    if templates_sel[img_idx] == 1:
        return templates1
    elif templates_sel[img_idx] == 3:
        return templates3
    elif templates_sel[img_idx] == 5:
        return templates5
    else:
        return templates1

similarities_all = []
converted_imgs_all = []
print('  Contour No. ', end='')
for i,target_ctr in enumerate(ctr_datasets_all):
    templates = select_template(i)
    print(i, ' ', end='')
    sims, imgs = get_similarities(target_ctr, templates)
    similarities_all += [sims]
    converted_imgs_all += [imgs]

  Contour No. 0  1  2  3  4  5  6  7  8  9  10  11  12  13  14  15  16  17  18  19  20  21  22  23  24  25  26  27  28  29  30  31  32  33  34  35  36  37  38  39  40  41  42  43  44  45  46  47  48  49  50  51  52  53  54  55  56  57  58  59  60  61  62  63  64  65  66  67  68  69  70  71  72  73  74  75  76  77  78  79  80  81  82  83  84  85  86  87  88  89  90  91  92  93  94  95  96  97  98  99  100  101  102  103  104  105  106  107  108  109  110  111  112  113  114  115  116  117  118  119  120  121  122  123  124  125  126  127  128  129  130  131  132  133  134  135  136  137  138  139  140  141  142  143  144  145  146  147  148  149  150  151  152  153  154  155  156  157  158  159  160  161  162  163  164  165  166  167  168  169  170  171  172  173  174  175  176  177  178  179  180  181  182  183  184  185  186  187  188  189  190  191  192  193  194  195  196  197  198  199  200  201  202  203  204  205  206  207  208  209  210  211  212  213  214  215  216  217  218  219  220  221  222  223  224  225  226  227  228  229  230  231  232  233  234  235  236  237  238  239  240  241  242  243  244  245  246  247  248  249  250  251  252  253  254  255  256  257  258  259  260  261  262  263  264  

得られた一致度を全て表示しておきます。
長くなっちゃいますが…

for sim, lab in zip(similarities_all, labels_all):
    print('label {:>2}'.format(lab), ': ', end='')
    print_similarity_vector(sim, end='\n')

label -1 : [0.825, 0.879, 0.666, 0.649, 0.710, ]
label -1 : [0.820, 0.783, 0.658, 0.667, 0.699, ]
label -1 : [0.841, 0.757, 0.684, 0.676, 0.733, ]
label -1 : [0.816, 0.730, 0.676, 0.699, 0.731, ]
label -1 : [0.844, 0.765, 0.691, 0.708, 0.738, ]
label -1 : [0.860, 0.746, 0.697, 0.681, 0.736, ]
label  5 : [0.717, 0.816, 0.685, 0.812, 0.919, ]
label  0 : [0.868, 0.836, 0.608, 0.679, 0.731, ]
label  5 : [0.746, 0.748, 0.672, 0.792, 0.950, ]
label  1 : [0.729, 0.942, 0.766, 0.743, 0.731, ]
label  5 : [0.731, 0.774, 0.668, 0.781, 0.921, ]
label  0 : [0.942, 0.762, 0.695, 0.669, 0.728, ]
label  2 : [0.660, 0.753, 0.937, 0.754, 0.724, ]
label  1 : [0.744, 0.947, 0.710, 0.702, 0.686, ]
label  2 : [0.650, 0.777, 0.957, 0.754, 0.702, ]
label -1 : [0.682, 0.747, 0.643, 0.630, 0.647, ]
label -1 : [0.796, 0.672, 0.715, 0.783, 0.878, ]
label  1 : [0.709, 0.952, 0.801, 0.784, 0.755, ]
label  1 : [0.625, 0.955, 0.834, 0.807, 0.805, ]
label  5 : [0.732, 0.759, 0.663, 0.786, 0.918, ]
label  0 : [0.967, 0.722, 0.670, 0.659, 0.732, ]
label  2 : [0.665, 0.715, 0.945, 0.751, 0.682, ]
label  5 : [0.719, 0.680, 0.691, 0.801, 0.898, ]
label  0 : [0.963, 0.722, 0.678, 0.664, 0.728, ]
label  2 : [0.670, 0.732, 0.963, 0.754, 0.697, ]
label  5 : [0.750, 0.693, 0.661, 0.674, 0.913, ]
label  0 : [0.895, 0.734, 0.684, 0.629, 0.752, ]
label  1 : [0.744, 0.954, 0.740, 0.708, 0.706, ]
label  2 : [0.681, 0.720, 0.942, 0.750, 0.692, ]
label -1 : [0.820, 0.738, 0.691, 0.713, 0.750, ]
label  5 : [0.671, 0.761, 0.653, 0.610, 0.784, ]
label  1 : [0.699, 0.953, 0.711, 0.688, 0.718, ]
label  2 : [0.621, 0.812, 0.958, 0.745, 0.736, ]
label  3 : [0.701, 0.664, 0.773, 1.000, 0.669, ]
label  1 : [0.542, 0.959, 0.837, 0.809, 0.818, ]
label  5 : [0.737, 0.728, 0.704, 0.782, 0.924, ]
label  0 : [0.865, 0.806, 0.655, 0.669, 0.721, ]
label -1 : [0.690, 0.781, 0.546, 0.677, 0.695, ]
label -1 : [0.840, 0.879, 0.677, 0.662, 0.765, ]
label -1 : [0.827, 0.787, 0.696, 0.688, 0.737, ]
label -1 : [0.844, 0.770, 0.645, 0.645, 0.705, ]
label -1 : [0.829, 0.750, 0.676, 0.751, 0.701, ]
label -1 : [0.862, 0.735, 0.684, 0.691, 0.728, ]
label -1 : [0.855, 0.758, 0.664, 0.664, 0.732, ]
label  5 : [0.709, 0.806, 0.687, 0.792, 0.901, ]
label  0 : [0.879, 0.821, 0.640, 0.667, 0.724, ]
label  5 : [0.734, 0.762, 0.677, 0.706, 0.928, ]
label  1 : [0.728, 0.941, 0.771, 0.757, 0.753, ]
label  5 : [0.726, 0.777, 0.675, 0.730, 0.931, ]
label  0 : [0.932, 0.782, 0.628, 0.660, 0.724, ]
label  2 : [0.657, 0.751, 0.946, 0.752, 0.731, ]
label  1 : [0.748, 0.936, 0.737, 0.708, 0.705, ]
label  2 : [0.636, 0.782, 0.956, 0.758, 0.714, ]
label -1 : [0.780, 0.648, 0.735, 0.787, 0.825, ]
label -1 : [0.687, 0.727, 0.642, 0.646, 0.660, ]
label  1 : [0.708, 0.943, 0.796, 0.778, 0.749, ]
label  1 : [0.618, 0.952, 0.836, 0.811, 0.812, ]
label  5 : [0.734, 0.775, 0.682, 0.785, 0.920, ]
label  0 : [0.962, 0.708, 0.666, 0.664, 0.727, ]
label -1 : [0.778, 0.783, 0.687, 0.701, 0.737, ]
label  2 : [0.663, 0.722, 0.943, 0.748, 0.681, ]
label  5 : [0.739, 0.666, 0.692, 0.812, 0.904, ]
label  0 : [0.962, 0.704, 0.672, 0.665, 0.724, ]
label  2 : [0.668, 0.736, 0.951, 0.746, 0.700, ]
label  5 : [0.750, 0.693, 0.653, 0.675, 0.922, ]
label  0 : [0.905, 0.707, 0.662, 0.630, 0.752, ]
label  1 : [0.745, 0.940, 0.738, 0.712, 0.701, ]
label  2 : [0.685, 0.718, 0.946, 0.755, 0.680, ]
label  5 : [0.750, 0.675, 0.710, 0.794, 0.915, ]
label  0 : [0.827, 0.785, 0.669, 0.627, 0.743, ]
label  1 : [0.695, 0.945, 0.717, 0.692, 0.705, ]
label -1 : [0.775, 0.859, 0.708, 0.691, 0.725, ]
label  2 : [0.610, 0.814, 0.959, 0.748, 0.742, ]
label -1 : [0.798, 0.863, 0.684, 0.697, 0.788, ]
label -1 : [0.805, 0.851, 0.696, 0.675, 0.733, ]
label -1 : [0.786, 0.765, 0.675, 0.659, 0.686, ]
label  3 : [0.708, 0.671, 0.767, 0.969, 0.668, ]
label -1 : [0.942, 0.719, 0.693, 0.695, 0.736, ]
label  5 : [0.750, 0.737, 0.693, 0.781, 0.897, ]
label  0 : [0.845, 0.789, 0.660, 0.674, 0.745, ]
label -1 : [0.802, 0.805, 0.702, 0.701, 0.749, ]
label  1 : [0.533, 0.944, 0.835, 0.813, 0.818, ]
label -1 : [0.796, 0.821, 0.687, 0.696, 0.698, ]
label -1 : [0.776, 0.824, 0.753, 0.714, 0.738, ]
label -1 : [0.824, 0.699, 0.615, 0.639, 0.693, ]
label -1 : [0.694, 0.682, 0.633, 0.664, 0.656, ]
label -1 : [0.868, 0.735, 0.673, 0.691, 0.761, ]
label -1 : [0.825, 0.735, 0.673, 0.680, 0.761, ]
label  1 : [0.691, 0.954, 0.762, 0.784, 0.729, ]
label  1 : [0.668, 0.947, 0.743, 0.743, 0.731, ]
label  5 : [0.765, 0.711, 0.683, 0.706, 0.938, ]
label  0 : [1.000, 0.710, 0.650, 0.628, 0.756, ]
label  1 : [0.646, 0.947, 0.714, 0.707, 0.692, ]
label  1 : [0.666, 0.945, 0.743, 0.732, 0.731, ]
label  5 : [0.679, 0.768, 0.687, 0.752, 0.929, ]
label  0 : [0.843, 0.753, 0.688, 0.635, 0.697, ]
label  5 : [0.759, 0.717, 0.690, 0.681, 0.934, ]
label  0 : [0.956, 0.690, 0.633, 0.625, 0.690, ]
label -1 : [0.825, 0.811, 0.735, 0.761, 0.692, ]
label -1 : [0.811, 0.700, 0.636, 0.668, 0.719, ]
label -1 : [0.793, 0.667, 0.632, 0.641, 0.730, ]
label  2 : [0.650, 0.785, 0.979, 0.762, 0.705, ]
label -1 : [0.784, 0.751, 0.686, 0.691, 0.701, ]
label -1 : [0.847, 0.729, 0.668, 0.679, 0.733, ]
label -1 : [0.808, 0.697, 0.650, 0.660, 0.719, ]
label  1 : [0.741, 0.940, 0.738, 0.715, 0.674, ]
label  1 : [0.671, 0.944, 0.730, 0.717, 0.691, ]
label  1 : [0.670, 0.941, 0.730, 0.721, 0.698, ]
label -1 : [0.797, 0.747, 0.689, 0.692, 0.734, ]
label  1 : [0.685, 0.945, 0.718, 0.695, 0.685, ]
label -1 : [0.822, 0.713, 0.644, 0.692, 0.706, ]
label -1 : [0.814, 0.667, 0.624, 0.650, 0.726, ]
label  1 : [0.706, 0.952, 0.716, 0.713, 0.690, ]
label  1 : [0.639, 0.954, 0.687, 0.689, 0.674, ]
label -1 : [0.798, 0.671, 0.620, 0.654, 0.749, ]
label  2 : [0.662, 0.711, 0.951, 0.753, 0.651, ]
label -1 : [0.811, 0.718, 0.688, 0.723, 0.721, ]
label  1 : [0.639, 0.960, 0.683, 0.690, 0.674, ]
label -1 : [0.802, 0.692, 0.615, 0.658, 0.731, ]
label  1 : [0.702, 0.948, 0.726, 0.715, 0.678, ]
label  2 : [0.601, 0.755, 0.944, 0.754, 0.676, ]
label -1 : [0.819, 0.714, 0.647, 0.730, 0.719, ]
label  1 : [0.677, 0.941, 0.718, 0.706, 0.692, ]
label -1 : [0.823, 0.684, 0.640, 0.672, 0.744, ]
label -1 : [0.648, 0.847, 0.697, 0.666, 0.681, ]
label  2 : [0.592, 0.757, 0.965, 0.757, 0.697, ]
label  5 : [0.753, 0.692, 0.683, 0.684, 0.925, ]
label -1 : [0.648, 0.762, 0.662, 0.664, 0.762, ]
label  0 : [0.956, 0.696, 0.640, 0.629, 0.703, ]
label -1 : [0.811, 0.688, 0.637, 0.671, 0.753, ]
label  1 : [0.678, 0.944, 0.702, 0.689, 0.690, ]
label  1 : [0.656, 0.949, 0.684, 0.703, 0.679, ]
label -1 : [0.561, 0.841, 0.807, 0.802, 0.845, ]
label -1 : [0.791, 0.805, 0.721, 0.730, 0.730, ]
label -1 : [0.846, 0.798, 0.613, 0.641, 0.669, ]
label -1 : [0.797, 0.786, 0.683, 0.681, 0.721, ]
label -1 : [0.849, 0.748, 0.700, 0.676, 0.739, ]
label -1 : [0.867, 0.731, 0.655, 0.648, 0.766, ]
label -1 : [0.874, 0.761, 0.672, 0.641, 0.717, ]
label -1 : [0.833, 0.817, 0.648, 0.674, 0.684, ]
label -1 : [0.908, 0.825, 0.694, 0.703, 0.659, ]
label -1 : [0.950, 0.822, 0.667, 0.693, 0.710, ]
label -1 : [0.894, 0.812, 0.611, 0.679, 0.680, ]
label -1 : [0.854, 0.730, 0.641, 0.644, 0.737, ]
label -1 : [0.894, 0.801, 0.696, 0.696, 0.714, ]
label -1 : [0.806, 0.769, 0.694, 0.658, 0.679, ]
label -1 : [0.908, 0.816, 0.677, 0.696, 0.766, ]
label -1 : [0.700, 0.706, 0.656, 0.650, 0.686, ]
label -1 : [0.804, 0.709, 0.707, 0.770, 0.848, ]
label -1 : [0.749, 0.721, 0.664, 0.695, 0.740, ]
label  1 : [0.706, 0.948, 0.771, 0.721, 0.749, ]
label  1 : [0.684, 0.959, 0.769, 0.728, 0.740, ]
label  1 : [0.658, 0.955, 0.807, 0.763, 0.778, ]
label  1 : [0.641, 0.958, 0.837, 0.798, 0.790, ]
label  1 : [0.512, 0.966, 0.874, 0.829, 0.841, ]
label  1 : [0.696, 0.948, 0.776, 0.726, 0.749, ]
label  1 : [0.684, 0.950, 0.773, 0.722, 0.742, ]
label -1 : [0.502, 0.738, 0.779, 0.647, 0.712, ]
label  5 : [0.731, 0.688, 0.678, 0.792, 0.920, ]
label  0 : [0.931, 0.718, 0.646, 0.630, 0.701, ]
label  2 : [0.652, 0.753, 0.936, 0.755, 0.659, ]
label  5 : [0.738, 0.712, 0.653, 0.688, 0.913, ]
label  0 : [0.904, 0.825, 0.639, 0.635, 0.716, ]
label  2 : [0.653, 0.744, 0.935, 0.749, 0.660, ]
label  1 : [0.639, 0.950, 0.806, 0.758, 0.779, ]
label  2 : [0.640, 0.760, 0.930, 0.750, 0.674, ]
label  2 : [0.632, 0.763, 0.933, 0.751, 0.670, ]
label -1 : [0.639, 0.866, 0.668, 0.667, 0.713, ]
label -1 : [0.593, 0.783, 0.596, 0.620, 0.762, ]
label -1 : [0.524, 0.772, 0.683, 0.692, 0.714, ]
label  1 : [0.698, 0.948, 0.756, 0.710, 0.746, ]
label  1 : [0.765, 0.947, 0.749, 0.705, 0.716, ]
label  1 : [0.667, 0.950, 0.791, 0.732, 0.762, ]
label -1 : [0.826, 0.729, 0.671, 0.645, 0.703, ]
label -1 : [0.827, 0.738, 0.665, 0.650, 0.723, ]
label  2 : [0.628, 0.794, 0.939, 0.756, 0.670, ]
label  0 : [0.812, 0.814, 0.643, 0.620, 0.702, ]
label  1 : [0.556, 0.970, 0.867, 0.825, 0.738, ]
label  5 : [0.639, 0.775, 0.670, 0.784, 0.964, ]
label -1 : [0.473, 0.880, 0.593, 0.620, 0.661, ]
label  1 : [0.659, 0.959, 0.840, 0.745, 0.738, ]
label  1 : [0.770, 0.939, 0.776, 0.718, 0.738, ]
label  1 : [0.704, 0.951, 0.778, 0.726, 0.737, ]
label  1 : [0.713, 0.949, 0.784, 0.748, 0.733, ]
label  1 : [0.637, 0.951, 0.816, 0.756, 0.723, ]
label  1 : [0.701, 0.945, 0.831, 0.750, 0.752, ]
label  1 : [0.511, 0.955, 0.867, 0.829, 0.835, ]
label  1 : [0.708, 0.930, 0.788, 0.743, 0.734, ]
label  1 : [0.674, 0.951, 0.804, 0.724, 0.743, ]
label -1 : [0.909, 0.806, 0.682, 0.671, 0.690, ]
label  5 : [0.624, 0.674, 0.638, 0.662, 0.919, ]
label  1 : [0.636, 0.946, 0.806, 0.763, 0.733, ]
label  0 : [0.840, 0.820, 0.620, 0.627, 0.713, ]
label  5 : [0.710, 0.685, 0.683, 0.681, 0.912, ]
label  1 : [0.722, 0.942, 0.823, 0.780, 0.737, ]
label  2 : [0.633, 0.760, 0.939, 0.767, 0.675, ]
label  0 : [0.925, 0.727, 0.645, 0.649, 0.690, ]
label  5 : [0.780, 0.722, 0.641, 0.661, 0.931, ]
label  0 : [0.932, 0.750, 0.656, 0.633, 0.716, ]
label  2 : [0.653, 0.723, 0.938, 0.762, 0.641, ]
label  1 : [0.667, 0.917, 0.815, 0.761, 0.734, ]
label  2 : [0.693, 0.757, 0.922, 0.744, 0.687, ]
label  2 : [0.675, 0.738, 0.931, 0.753, 0.655, ]
label -1 : [0.581, 0.792, 0.594, 0.588, 0.699, ]
label -1 : [0.578, 0.823, 0.707, 0.668, 0.700, ]
label  1 : [0.704, 0.957, 0.713, 0.668, 0.743, ]
label  1 : [0.784, 0.927, 0.759, 0.718, 0.705, ]
label  1 : [0.728, 0.948, 0.756, 0.704, 0.751, ]
label -1 : [0.748, 0.825, 0.723, 0.730, 0.782, ]
label  0 : [0.887, 0.816, 0.638, 0.672, 0.681, ]
label  1 : [0.506, 0.960, 0.864, 0.842, 0.852, ]
label  5 : [0.704, 0.795, 0.679, 0.799, 0.926, ]
label  2 : [0.588, 0.763, 0.923, 0.752, 0.676, ]
label -1 : [0.507, 0.676, 0.625, 0.583, 0.567, ]
label  1 : [0.563, 0.922, 0.856, 0.820, 0.832, ]
label  1 : [0.727, 0.938, 0.820, 0.791, 0.779, ]
label  1 : [0.748, 0.937, 0.736, 0.714, 0.754, ]
label  1 : [0.751, 0.937, 0.763, 0.728, 0.775, ]
label  5 : [0.608, 0.700, 0.751, 0.711, 0.911, ]
label  1 : [0.729, 0.879, 0.789, 0.756, 0.684, ]
label  0 : [0.786, 0.820, 0.623, 0.615, 0.703, ]
label  5 : [0.677, 0.712, 0.638, 0.816, 0.885, ]
label  0 : [0.881, 0.804, 0.612, 0.624, 0.705, ]
label  2 : [0.615, 0.789, 0.944, 0.771, 0.665, ]
label  1 : [0.605, 0.951, 0.846, 0.807, 0.809, ]
label  5 : [0.706, 0.694, 0.668, 0.698, 0.912, ]
label  0 : [0.929, 0.736, 0.664, 0.646, 0.695, ]
label  2 : [0.625, 0.736, 0.918, 0.763, 0.659, ]
label  2 : [0.674, 0.736, 0.919, 0.777, 0.657, ]
label  2 : [0.672, 0.740, 0.930, 0.746, 0.667, ]
label  1 : [0.706, 0.941, 0.765, 0.729, 0.751, ]
label -1 : [0.566, 0.826, 0.604, 0.581, 0.680, ]
label -1 : [0.587, 0.828, 0.695, 0.674, 0.676, ]
label  1 : [0.758, 0.884, 0.736, 0.702, 0.679, ]
label  1 : [0.784, 0.925, 0.756, 0.736, 0.742, ]
label  1 : [0.750, 0.933, 0.714, 0.690, 0.746, ]
label  1 : [0.709, 0.925, 0.799, 0.764, 0.711, ]
label  1 : [0.567, 0.959, 0.852, 0.809, 0.740, ]
label  1 : [0.578, 0.938, 0.850, 0.814, 0.822, ]
label  1 : [0.767, 0.938, 0.717, 0.696, 0.772, ]
label  1 : [0.727, 0.935, 0.768, 0.739, 0.744, ]
label -1 : [0.851, 0.864, 0.698, 0.668, 0.646, ]
label -1 : [0.811, 0.783, 0.728, 0.724, 0.762, ]
label -1 : [0.960, 0.817, 0.667, 0.725, 0.724, ]
label -1 : [0.818, 0.827, 0.724, 0.692, 0.776, ]
label -1 : [0.796, 0.750, 0.694, 0.659, 0.664, ]
label -1 : [0.872, 0.776, 0.711, 0.697, 0.678, ]
label  1 : [0.755, 0.939, 0.801, 0.753, 0.738, ]
label  2 : [0.664, 0.759, 0.933, 0.752, 0.686, ]
label  2 : [0.645, 0.762, 0.933, 0.756, 0.683, ]
label  2 : [0.669, 0.760, 0.934, 0.752, 0.686, ]
label  2 : [0.640, 0.794, 0.932, 0.764, 0.669, ]
label  1 : [0.726, 0.946, 0.722, 0.670, 0.716, ]
label  2 : [0.654, 0.742, 0.931, 0.767, 0.680, ]
label  2 : [0.606, 0.841, 0.929, 0.750, 0.658, ]
label  2 : [0.587, 0.841, 0.922, 0.766, 0.691, ]
label  1 : [0.680, 0.941, 0.838, 0.783, 0.784, ]
label -1 : [0.557, 0.829, 0.675, 0.598, 0.679, ]
label -1 : [0.509, 0.771, 0.691, 0.645, 0.692, ]
label -1 : [0.518, 0.814, 0.723, 0.639, 0.694, ]
label  2 : [0.637, 0.800, 0.935, 0.765, 0.670, ]
label  1 : [0.736, 0.926, 0.743, 0.700, 0.743, ]
label  2 : [0.658, 0.780, 0.949, 0.760, 0.672, ]
label  1 : [0.738, 0.936, 0.825, 0.778, 0.768, ]
label  1 : [0.728, 0.938, 0.718, 0.671, 0.730, ]

SVM学習実施

SVMの学習を実施します。
結果が良ければ、学習したSVMモデルを保存、今後使用します。
30番の輪郭は輪郭検出がうまくいっていないのが分かっているので削除しておきます。

svm_inputs = copy.deepcopy(similarities_all)
svm_labels = copy.deepcopy(labels_all)

# Remove inadequate contour data in img1
del svm_inputs[30]
del svm_labels[30]

train_data, train_labels = get_random_sample(svm_inputs, svm_labels, [-1,0,1,2,3,5], 20, seed=123)
svm = prepare_svm(train_data, train_labels)

SVM推論実施

学習したSVMモデルの性能を確認。

result = svm.predict(np.array(svm_inputs, 'float32'))
print_stat(result, svm_labels)

label -1 : {-1: 73, 0: 13, 1:  0, 2:  0, 3:  0, 5:  3, }
label  0 : {-1:  5, 0: 22, 1:  0, 2:  0, 3:  0, 5:  0, }
label  1 : {-1:  0, 0:  0, 1: 78, 2:  0, 3:  0, 5:  0, }
label  2 : {-1:  0, 0:  0, 1:  0, 2: 39, 3:  0, 5:  0, }
label  3 : {-1:  0, 0:  0, 1:  0, 2:  0, 3:  2, 5:  0, }
label  5 : {-1:  0, 0:  0, 1:  0, 2:  0, 3:  0, 5: 29, }

"0"以外の数字の輪郭であれば全て正解していますが、非数字の輪郭で判定の失敗がいくつかあります。
判定に失敗した輪郭を確認します。

subimgs = copy.deepcopy(subimgs_all)
subctrs = copy.deepcopy(subctrs_all)

del subimgs[30]
del subctrs[30]

for i,(sims,lab,res,img,ctr) in enumerate(zip(svm_inputs, svm_labels, result[1], subimgs, subctrs)):
    if lab != res[0]:
        print('No.', i)
        print('{: }'.format(lab), ' -> ', '{: d}'.format(int(res[0])), ' [',end='')
        for s in sims: print('{:.3f}, '.format(s), end='');
        print(']')
        img = cv2.drawContours(img, [ctr], -1, (0,255,0), 1)
        plt.imshow(cv2.cvtColor(img, cv2.COLOR_BGR2RGB)),plt.xticks([]),plt.yticks([])
        plt.show()

    No. 16
    -1  ->   5  [0.796, 0.672, 0.715, 0.783, 0.878, ]

    No. 37
    -1  ->   0  [0.840, 0.879, 0.677, 0.662, 0.765, ]

    No. 52
    -1  ->   5  [0.780, 0.648, 0.735, 0.787, 0.825, ]

    No. 68
     0  ->  -1  [0.827, 0.785, 0.669, 0.627, 0.743, ]

    No. 76
    -1  ->   0  [0.942, 0.719, 0.693, 0.695, 0.736, ]

    No. 78
     0  ->  -1  [0.845, 0.789, 0.660, 0.674, 0.745, ]

    No. 94
     0  ->  -1  [0.843, 0.753, 0.688, 0.635, 0.697, ]

    No. 133
    -1  ->   0  [0.846, 0.798, 0.613, 0.641, 0.669, ]

    No. 136
    -1  ->   0  [0.867, 0.731, 0.655, 0.648, 0.766, ]

    No. 137
    -1  ->   0  [0.874, 0.761, 0.672, 0.641, 0.717, ]

    No. 139
    -1  ->   0  [0.908, 0.825, 0.694, 0.703, 0.659, ]

    No. 140
    -1  ->   0  [0.950, 0.822, 0.667, 0.693, 0.710, ]

    No. 141
    -1  ->   0  [0.894, 0.812, 0.611, 0.679, 0.680, ]

    No. 142
    -1  ->   0  [0.854, 0.730, 0.641, 0.644, 0.737, ]

    No. 143
    -1  ->   0  [0.894, 0.801, 0.696, 0.696, 0.714, ]

    No. 145
    -1  ->   0  [0.908, 0.816, 0.677, 0.696, 0.766, ]

    No. 147
    -1  ->   5  [0.804, 0.709, 0.707, 0.770, 0.848, ]

    No. 175
     0  ->  -1  [0.812, 0.814, 0.643, 0.620, 0.702, ]

    No. 188
    -1  ->   0  [0.909, 0.806, 0.682, 0.671, 0.690, ]

    No. 219
     0  ->  -1  [0.786, 0.820, 0.623, 0.615, 0.703, ]

    No. 242
    -1  ->   0  [0.960, 0.817, 0.667, 0.725, 0.724, ]

SVM再学習

判定に失敗した輪郭データの一部を学習データに加えます。
点数文字でない"5"が3つ見られますが、このうち2つを追加します。
その後再度判定を行ってみます。

train_data += [svm_inputs[16], svm_inputs[52]]
train_labels += [svm_labels[16], svm_labels[52]]
svm = prepare_svm(train_data, train_labels)

result = svm.predict(np.array(svm_inputs, 'float32'))
print_stat(result, svm_labels)

label -1 : {-1: 75, 0: 13, 1:  0, 2:  0, 3:  0, 5:  1, }
label  0 : {-1:  5, 0: 22, 1:  0, 2:  0, 3:  0, 5:  0, }
label  1 : {-1:  0, 0:  0, 1: 78, 2:  0, 3:  0, 5:  0, }
label  2 : {-1:  0, 0:  0, 1:  0, 2: 39, 3:  0, 5:  0, }
label  3 : {-1:  0, 0:  0, 1:  0, 2:  0, 3:  2, 5:  0, }
label  5 : {-1:  0, 0:  0, 1:  0, 2:  0, 3:  0, 5: 29, }

"5"の文字を全て正しく判別できるかと期待したが、そうはならず。
このまま進めてしまうこととします。

データ化

ここまでで得られたテンプレートデータとSVMモデルデータを保存し、点数計算アプリで使用したい。

SVMモデルの保存

SVMは保存するメソッドがあったので、それを使用します。
テンプレートデータについては、pythonでのオブジェクト保存の方法を調べるとpickle、jsonを使った2パターンが出てきます。 pickleでは、出所の分からないデータの読み込みを行うと意図しないコードを実行されてしまう、という脆弱性があるようですが、今回は自分で用意したデータを使いたいだけなので、一応問題はないかなと。
ただ、jsonを知っておいたほうが今後のためになりそうなので、jsonを使います。

Python公式 pickleドキュメント:

https://docs.python.org/ja/3/library/pickle.html

Python公式 jsonドキュメント:

https://docs.python.org/ja/3/library/json.html

Pythonでのjson参考:

https://hibiki-press.tech/python/json/1633
https://note.nkmk.me/python-json-load-dump/

svm.save('harupan_data/harupan_svm.dat')

生成されたデータの中身はこんな感じ。
テキストで、設定した内容も含めてSVMの情報が全部入っています。

%YAML:1.0
---
opencv_ml_svm:
   format: 3
   svmType: C_SVC
   kernel:
      type: LINEAR
   C: 100.
   term_criteria: { epsilon:1.1920928955078125e-07, iterations:1000 }
   var_count: 5
   class_count: 6
   class_labels: !!opencv-matrix
      rows: 6
      cols: 1
      dt: i
      data: [ -1, 0, 1, 2, 3, 5 ]
   sv_total: 15
   support_vectors:
      - [ -1.80685959e+01, -5.69025517e+00, 1.04596977e+01,
          7.75204945e+00, -2.17070246e+00 ]
      - [ 1.23291440e-01, -1.82865601e+01, -6.46431494e+00,
          4.22870070e-01, 6.76044846e+00 ]

...

      - [ -2.71152169e-01, -2.04700381e-01, 1.73537850e+00,
          3.85429978e+00, -5.29199266e+00 ]
   uncompressed_sv_total: 38
   uncompressed_support_vectors:
      - [ 8.49453330e-01, 7.48484850e-01, 6.99999988e-01, 6.75757587e-01,
          7.39393950e-01 ]
      - [ 8.43867898e-01, 7.64705896e-01, 6.90823317e-01, 7.08056450e-01,
          7.38181829e-01 ]
          
...
          
      - [ 7.79716969e-01, 6.48148119e-01, 7.35420227e-01, 7.86544859e-01,
          8.25454533e-01 ]
   decision_functions:
      -
         sv_count: 1
         rho: -9.4865848064106029e+00
         alpha: [ 1. ]
         index: [ 0 ]
      -
         sv_count: 1
         rho: -1.5486015742808821e+01
         alpha: [ 1. ]
         index: [ 1 ]
      -
         sv_count: 1
         rho: 5.0411393634237656e-01
         alpha: [ 1. ]
         index: [ 2 ]
         
...
         
      -
         sv_count: 1
         rho: 2.3617392745496142e+00
         alpha: [ 1. ]
         index: [ 13 ]
      -
         sv_count: 1
         rho: 1.9971712154702148e-01
         alpha: [ 1. ]
         index: [ 14 ]

保存データを読み込んで確認してみます。
Pythonでのモデル読み込みは以下を参照。

https://qiita.com/color_box/items/6f7d06fc3d65c6913ebf

svm_restored = cv2.ml.SVM_load('harupan_data/harupan_svm.dat')
result = svm_restored.predict(np.array(svm_inputs, 'float32'))
print_stat(result, svm_labels)

label -1 : {-1: 75, 0: 13, 1:  0, 2:  0, 3:  0, 5:  1, }
label  0 : {-1:  5, 0: 22, 1:  0, 2:  0, 3:  0, 5:  0, }
label  1 : {-1:  0, 0:  0, 1: 78, 2:  0, 3:  0, 5:  0, }
label  2 : {-1:  0, 0:  0, 1:  0, 2: 39, 3:  0, 5:  0, }
label  3 : {-1:  0, 0:  0, 1:  0, 2:  0, 3:  2, 5:  0, }
label  5 : {-1:  0, 0:  0, 1:  0, 2:  0, 3:  0, 5: 29, }

前と同じ結果が出ているので、問題なし。

テンプレートデータの保存(json)

テンプレートデータのjson保存のほうは、ndarrayがそのままでは扱えなかったので、リストに一回変換してから保存します。
読み込みのときに、ndarrayに戻します。

また、辞書データはjson形式と相性がいいようなので、これも組み合わせ。

https://wtnvenga.hatenablog.com/entry/2018/05/27/113848
https://note.nkmk.me/python-numpy-list/

テンプレートデータは、輪郭データと選択点リストを保存しておくこととします。
下準備の段階で、これらのデータを上に書いたクラスに与えてインスタンスを生成します。

import json

# ctr_list: List of contours for (0, 1, 2, 3, 5)
# pts_idx_list: List of selected point indices for (1, 2, 3, 5)
def save_templates(filename, ctr_list, pts_idx_list):
    with open(filename, mode='w') as f:
        save_data = []
        save_data += [{'num': 0, 'ctr': ctr_list[0].tolist(), 'pts': [0]}]
        for num, ctr, pts_idx in zip([1,2,3,5], ctr_list[1:5], pts_idx_list):
            save_data += [{'num': num, 'ctr': ctr.tolist(), 'pts': pts_idx}]
        json.dump(save_data, f, indent=2)
    return

def load_templates(filename):
    with open(filename, mode='r') as f:
        load_data = json.load(f)
        templates_rtn = []
        for d in load_data:
            templates_rtn += [template_dataset(np.array(d['ctr']), d['num'], d['pts'])]
    return templates_rtn

save_templates('harupan_data/templates2019.json', subctrs1, subctrs1_selected_pts)
save_templates('harupan_data/templates2020.json', subctrs3, subctrs3_selected_pts)
save_templates('harupan_data/templates2021.json', subctrs5, subctrs5_selected_pts)

生成されたデータは以下のような形。

[
  {
    "num": 0,
    "ctr": [
      [
        [
          18,
          52
        ]
      ],
      [
        [
          16,
          52
        ]
      ],
      [
        [
          15,
          52
        ]
      ],

...

      [
        [
          19,
          52
        ]
      ],
      [
        [
          19,
          52
        ]
      ]
    ],
    "pts": [
      0
    ]
  },
  {
    "num": 1,
    "ctr": [
      [
        [
          0,
          2
        ]
      ],
      [
        [
          1,
          0
        ]
      ],
      [
        [
          1,
          0
        ]
      ],

...

      [
        [
          0,
          4
        ]
      ],
      [
        [
          0,
          3
        ]
      ]
    ],
    "pts": [
      0,
      5,
      10,
      15,
      20,
      25,
      30,
      35,
      40,
      45,
      50,
      55,
      60,
      65,
      70,
      75,
      80,
      85,
      90,
      95,
      100,
      105,
      110,
      115,
      120,
      125,
      130,
      135,
      140
    ]
  },
  {
    "num": 2,
    "ctr": [
      [
        [
          12,
          1
        ]
      ],
      [
        [
          13,
          0
        ]
      ],

...

分かりやすくはないですが、直接見たいわけではないので特に問題なし。
テキスト保存なので無駄にデータ量が多くなりますが…。

きちんと復元できることも確認。

templates1_restored = load_templates('harupan_data/templates2019.json')
templates3_restored = load_templates('harupan_data/templates2020.json')
templates5_restored = load_templates('harupan_data/templates2021.json')

def disp_template(template):
    img = cv2.cvtColor(template.solid.copy(), cv2.COLOR_GRAY2RGB)
    if template.num != 0:
        img = cv2.drawContours(img, [template.ctr], -1, (0,255,0), 1)
        for p in template.pts:
            img = cv2.drawMarker(img, p, (255,0,0), markerType=cv2.MARKER_CROSS, markerSize=3)
    plt.imshow(img), plt.xticks([]), plt.yticks([])
    plt.show()

print('Template 2019')
for t in templates1_restored:
    disp_template(t)

print('Template 2020')
for t in templates3_restored:
    disp_template(t)

print('Template 2021')
for t in templates5_restored:
    disp_template(t)

    Template 2019

    Template 2020

    Template 2021

大丈夫そう。

スクリプト化

今までの処理を、pythonスクリプトとして保存、これを読み込めば必要な関数がロードされるようにしておきたいなと。

以下のスクリプト(harupan.py)を作成。
これでJupyter notebookがすっきりするか。

######################################################
# Importing libraries
######################################################
import cv2
import numpy as np
from matplotlib import pyplot as plt
import math
import copy
import random
import json

######################################################
# Detecting contours
######################################################
def detect_candidate_contours(image, res_th=800):
    h, w, chs = image.shape
    if h > res_th or w > res_th:
        k = float(res_th)/h if w > h else float(res_th)/w
    else:
        k = 1.0
    img = cv2.resize(image, None, fx=k, fy=k, interpolation=cv2.INTER_AREA)
    hsv = cv2.cvtColor(img, cv2.COLOR_BGR2HSV)
    # Convert hue value (rotation, mask by saturation)
    hsv[:,:,0] = np.where(hsv[:,:,0] < 50, hsv[:,:,0]+180, hsv[:,:,0])
    hsv[:,:,0] = np.where(hsv[:,:,1] < 100, 0, hsv[:,:,0])
    # Thresholding with cv2.inRange()
    th_hue = cv2.inRange(hsv[:,:,0], 135, 190)
    # Retrieve all points on the contours (cv2.CHAIN_APPROX_NONE)
    contours, hierarchy = cv2.findContours(th_hue, cv2.RETR_TREE, cv2.CHAIN_APPROX_NONE)
    indices0 = [i for i,hier in enumerate(hierarchy[0,:,:]) if hier[3] == -1]
    indices1 = [i for i,hier in enumerate(hierarchy[0,:,:]) if hier[3] in indices0]
    contours1 = [contours[i] for i in indices1]
    contours1_filtered = [ctr for ctr in contours1 if cv2.contourArea(ctr) > float(res_th)*float(res_th)/4000]
    return contours1_filtered, img


######################################################
# Auxiliary functions
######################################################
def create_contour_area_image(img, ctr):
    x,y,w,h = cv2.boundingRect(ctr)
    rtn_img = img[y:y+h,x:x+w,:].copy()
    rtn_ctr = ctr.copy()
    origin = np.array([x,y])
    for c in rtn_ctr:
        c[0,:] -= origin
    return rtn_img, rtn_ctr

# ctr: Should be output of create_contour_area_image() (Origin of points is the origin of bounding box)
# img_shape: Optional, tuple of (image_height, image_width), if omitted, calculated from ctr
def create_solid_contour(ctr, img_shape=(int(0),int(0))):
    if img_shape == (int(0),int(0)):
        _,_,w,h = cv2.boundingRect(ctr)
    else:
        h,w = img_shape
    img = np.zeros((h,w), 'uint8')
    img = cv2.drawContours(img, [ctr], -1, 255, -1)
    return img

# ctr: Should be output of create_contour_area_image() (Origin of points is the origin of bounding box)
def create_upright_solid_contour(ctr):
    (cx,cy),(w,h),angle = cv2.minAreaRect(ctr)
    M = cv2.getRotationMatrix2D((cx,cy), angle, 1)
    for i in range(ctr.shape[0]):
        ctr[i,0,:] = ( M @ np.array([ctr[i,0,0], ctr[i,0,1], 1]) ).astype('int')
    rect = cv2.boundingRect(ctr)
    img = np.zeros((rect[3],rect[2]), 'uint8')
    ctr -= rect[0:2]
    M[:,2] -= rect[0:2]
    img = cv2.drawContours(img, [ctr], -1, 255,-1)
    return img, M, ctr


######################################################
# Dataset classes
######################################################
class contour_dataset:
    def __init__(self, ctr):
        self.ctr = ctr.copy()
        self.rrect = cv2.minAreaRect(ctr)
        self.box = cv2.boxPoints(self.rrect)
        self.solid = create_solid_contour(ctr)
        self.pts = np.array([p for p in ctr[:,0,:]])

class template_dataset:
    def __init__(self, ctr, num, selected_idx=[0]):
        self.ctr = ctr.copy()
        self.num = num
        self.rrect = cv2.minAreaRect(ctr)
        self.box = cv2.boxPoints(self.rrect)
        if num == 0:
            self.solid,_,_ = create_upright_solid_contour(ctr)
        else:
            self.solid = create_solid_contour(ctr)
        self.pts = np.array([ctr[idx,0,:] for idx in selected_idx])


######################################################
# ICP
######################################################
# pts: list of 2D points, or ndarray of shape (n,2)
# query: 2D point to find nearest neighbor
def find_nearest_neighbor(pts, query):
    min_distance_sq = float('inf')
    min_idx = 0
    for i, p in enumerate(pts):
        d = np.dot(query - p, query - p)
        if(d < min_distance_sq):
            min_distance_sq = d
            min_idx = i
    return min_idx, np.sqrt(min_distance_sq)

# src, dst: ndarray, shape is (n,2) (n: number of points)
def estimate_affine_2d(src, dst):
    n = min(src.shape[0], dst.shape[0])
    x = dst[0:n].flatten()
    A = np.zeros((2*n,6))
    for i in range(n):
        A[i*2,0] = src[i,0]
        A[i*2,1] = src[i,1]
        A[i*2,2] = 1
        A[i*2+1,3] = src[i,0]
        A[i*2+1,4] = src[i,1]
        A[i*2+1,5] = 1
    M = np.linalg.inv(A.T @ A) @ A.T @ x
    return M.reshape([2,3])

# Find optimum affine matrix using ICP algorithm
# src_pts: ndarray, shape is (n_s,2) (n_s: number of points)
# dst_pts: ndarray, shape is (n_d,2) (n_d: number of points, n_d should be larger or equal to n_s)
# initial_matrix: ndarray, shape is (2,3)
def icp(src_pts, dst_pts, max_iter=20, initial_matrix=np.array([[1.0, 0.0, 0.0], [0.0, 1.0, 0.0]])):
    default_affine_matrix = np.array([[1.0, 0.0, 0.0], [0.0, 1.0, 0.0]])
    if dst_pts.shape[0] < src_pts.shape[0]:
        print("icp: Insufficient destination points")
        return default_affine_matrix, False
    if initial_matrix.shape != (2,3):
        print("icp: Illegal shape of initial_matrix")
        return default_affine_matrix, False
    M = initial_matrix
    # Store indices of the nearest neighbor point of dst_pts to the converted point of src_pts
    nn_idx = []
    for i in range(max_iter):
        nn_idx_tmp = []
        dst_pts_list = [p for p in dst_pts]
        idx_list = list(range(0,dst_pts.shape[0]))
        for p in src_pts:
            p2 = M @ np.array([p[0], p[1], 1])
            idx, d = find_nearest_neighbor(dst_pts_list, p2)
            nn_idx_tmp += [idx_list[idx]]
            del dst_pts_list[idx]
            del idx_list[idx]
        if nn_idx != [] and nn_idx == nn_idx_tmp:
            break
        dst_pts2 = np.zeros_like(src_pts)
        for j,idx in enumerate(nn_idx_tmp):
            dst_pts2[j,:] = dst_pts[idx,:]
        M = estimate_affine_2d(src_pts, dst_pts2)
        nn_idx = nn_idx_tmp
        if i == max_iter -1:
            return M, False
    return M, True


######################################################
# Calculating similarity and determining the number
######################################################
def binary_image_similarity(img1, img2):
    if img1.shape != img2.shape:
        print('binary_image_similarity: Different image size')
        return 0.0
    xor_img = cv2.bitwise_xor(img1, img2)
    return 1.0 - np.float(np.count_nonzero(xor_img)) / (img1.shape[0]*img2.shape[1])

# src, dst: contour_dataset or template_dataset (holding member variables box, solid)
def get_transform_by_rotated_rectangle(src, dst):
    # Rotated patterns are created when starting index is slided
    dst_box2 = np.vstack([dst.box, dst.box])
    max_similarity = 0.0
    max_converted_img = np.zeros((dst.solid.shape[1], dst.solid.shape[0]), 'uint8')
    for i in range(4):
        M = cv2.getAffineTransform(src.box[0:3], dst_box2[i:i+3])
        converted_img = cv2.warpAffine(src.solid, M, dsize=(dst.solid.shape[1], dst.solid.shape[0]), flags=cv2.INTER_NEAREST)
        similarity = binary_image_similarity(converted_img, dst.solid)
        if similarity > max_similarity:
            M_rtn = M
            max_similarity = similarity
            max_converted_img = converted_img
    return M_rtn, max_similarity, max_converted_img

def get_similarity_with_template(target_data, template_data, sim_th_high=0.95, sim_th_low=0.7):
    _,(w1,h1), _ = target_data.rrect
    _,(w2,h2), _ = template_data.rrect
    r = w1/h1 if w1 < h1 else h1/w1
    r = r * h2/w2 if w2 < h2 else r * w2/h2
    M, sim_init, _ = get_transform_by_rotated_rectangle(template_data, target_data)
    if sim_init > sim_th_high or sim_init < sim_th_low or r > 1.4 or r < 0.7:
        dsize = (template_data.solid.shape[1], template_data.solid.shape[0])
        flags = cv2.INTER_NEAREST|cv2.WARP_INVERSE_MAP
        converted_img = cv2.warpAffine(target_data.solid, M, dsize=dsize, flags=flags)
        return sim_init, converted_img
    M, _ = icp(template_data.pts, target_data.pts, initial_matrix=M)
    Minv = cv2.invertAffineTransform(M)
    converted_ctr = np.zeros_like(target_data.ctr)
    for i in range(target_data.ctr.shape[0]):
        converted_ctr[i,0,:] = (Minv[:,0:2] @ target_data.ctr[i,0,:]) + Minv[:,2]
    converted_img = create_solid_contour(converted_ctr, img_shape=template_data.solid.shape)
    val = binary_image_similarity(converted_img, template_data.solid)
    return val, converted_img

def get_similarity_with_template_zero(target_data, template_data):
    dsize = (template_data.solid.shape[1], template_data.solid.shape[0])
    converted_img = cv2.resize(target_data.solid, dsize=dsize, interpolation=cv2.INTER_NEAREST)
    val = binary_image_similarity(converted_img, template_data.solid)
    return val, converted_img

def get_similarities(target, templates):
    similarities = []
    converted_imgs = []
    for tmpl in templates:
        if tmpl.num == 0:
            sim,converted_img = get_similarity_with_template_zero(target, tmpl)
        else:
            sim,converted_img = get_similarity_with_template(target, tmpl)
        similarities += [sim]
        converted_imgs += [converted_img]
    return similarities, converted_imgs

# target: Single contour to compare
# templates: List of template_dataset (for numbers 0, 1, 2, 3, 5)
# svm: Trained SVM
# return: determined number (0,1,2,3,5), -1 if none corresponds
def determine_number(target, templates, svm):
    similarities,_ = get_similarities(target, templates)
    _, result = svm.predict(np.array(similarities))
    return int(result[0])


######################################################
# Loading template data and SVM model
######################################################
def load_svm(filename):
    return cv2.ml.SVM_load(filename)

def load_templates(filename):
    with open(filename, mode='r') as f:
        load_data = json.load(f)
        templates_rtn = []
        for d in load_data:
            templates_rtn += [template_dataset(np.array(d['ctr']), d['num'], d['pts'])]
    return templates_rtn

ここまで

春のパン祭りシール点数集計ではまだやらないといけないことがあって、

• 点数計算の処理
• 1枚の入力画像を受けてから点数を計算するまでの一連の流れ

というところですが、また次回にします。