OpenCVやってみる - 44. 処理の調整5(高速化)

春のパン祭り点数集計、前回で最後のつもりでしたが、1つ高速化の手段を思いついたので、もうちょっとがんばります。

変更内容

前に処理の様子を見た感じ、ICPアルゴリズムで時間がかかっているようでした。
テンプレートの輪郭点と対象輪郭の輪郭点間の最近傍点を探索するのに、輪郭上の全点間の距離を計算していたので、それが時間がかかったのかと。

思い付いたことは、

• 輪郭の点データは、輪郭の周に沿った順番に並んでいたので、全部の点からの探索をしなくてもいいのでは？

ということ。

処理の変更を実装して色々デバッグしましたが、なんとかうまくいったので、以下記載します。

下準備

春のパン祭り点数集計スクリプトの読み込み、テンプレートデータとSVMデータの読み込みを行います。
スクリプトのうち、ICP処理関数を変更します。

例によって、スクリプトファイル読み込みだと関数の上書きがうまくいかなかったので、以下に全部コピペします。
ブログが読みづらくなっちゃいますが…

スクリプト読み込み

前とほぼ同じスクリプトですが、calc_harupan()関数で、一致度ベクトルを表示させる変更が入っています。

######################################################
# Importing libraries
######################################################
from ctypes import resize
import cv2
import numpy as np
from matplotlib import pyplot as plt
import math
import copy
import random
import json

######################################################
# Detecting contours
######################################################
def reduce_resolution(img, res_th=800):
    h, w, chs = img.shape
    if h > res_th or w > res_th:
        k = float(res_th)/h if w > h else float(res_th)/w
    else:
        k = 1.0
    rtn_img = cv2.resize(img, None, fx=k, fy=k, interpolation=cv2.INTER_AREA)
    return rtn_img

def harupan_binarize(img, sat_th=100):
    hsv = cv2.cvtColor(img, cv2.COLOR_BGR2HSV)
    # Convert hue value (rotation, mask by saturation)
    hsv[:,:,0] = np.where(hsv[:,:,0] < 50, hsv[:,:,0]+180, hsv[:,:,0])
    hsv[:,:,0] = np.where(hsv[:,:,1] < sat_th, 0, hsv[:,:,0])
    # Thresholding with cv2.inRange()
    binary_img = cv2.inRange(hsv[:,:,0], 135, 190)
    return binary_img

def detect_candidate_contours(image, res_th=800, sat_th=100):
    img = reduce_resolution(image, res_th)
    binimg = harupan_binarize(img, sat_th)
    # Retrieve all points on the contours (cv2.CHAIN_APPROX_NONE)
    contours, hierarchy = cv2.findContours(binimg, cv2.RETR_TREE, cv2.CHAIN_APPROX_NONE)
    if len(contours) == 0:
        return contours, img
    # Pick up contours that have no parents
    indices = [i for i,hier in enumerate(hierarchy[0,:,:]) if hier[3] == -1]
    # Pick up contours that reside in above contours
    indices = [i for i,hier in enumerate(hierarchy[0,:,:]) if (hier[3] in indices) and (hier[2] == -1) ]
    contours = [contours[i] for i in indices]
    contours = [ctr for ctr in contours if cv2.contourArea(ctr) > float(res_th)*float(res_th)/4000]
    return contours, img

# image: Entire image containing multiple contours
# contours: Contours contained in "image" (Retrieved by cv2.findContours(), the origin is same as "image")
def refine_contours(image, contours):
    subctrs = []
    subimgs = []
    binimgs = []
    thresholds = []
    n_ctrs = []
    for ctr in contours:
        img, _ = create_contour_area_image(image, ctr)
        # Thresholding using G value in BGR format
        thresh, binimg = cv2.threshold(img[:,:,1], 0, 255, cv2.THRESH_BINARY + cv2.THRESH_OTSU)
        # Add black region around thresholded image, to detect contours correctly
        binimg = cv2.copyMakeBorder(binimg, 2,2,2,2, cv2.BORDER_CONSTANT, 0)
        ctrs2, _ = cv2.findContours(binimg, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_NONE)
        max_len = 0
        for ctr2 in ctrs2:
            if max_len <= ctr2.shape[0]:
                max_ctr = ctr2
                max_len = ctr2.shape[0]
        subctrs += [max_ctr]
        subimgs += [img]
        binimgs += [binimg]
        thresholds += [thresh]
        n_ctrs += [len(ctrs2)]
    debug_info = (binimgs, thresholds, n_ctrs)
    return subctrs, subimgs, debug_info

######################################################
# Auxiliary functions
######################################################
def create_contour_area_image(img, ctr):
    x,y,w,h = cv2.boundingRect(ctr)
    rtn_img = img[y:y+h,x:x+w,:].copy()
    rtn_ctr = ctr.copy()
    origin = np.array([x,y])
    for c in rtn_ctr:
        c[0,:] -= origin
    return rtn_img, rtn_ctr

# ctr: Should be output of create_contour_area_image() (Origin of points is the origin of bounding box)
# img_shape: Optional, tuple of (image_height, image_width), if omitted, calculated from ctr
def create_solid_contour(ctr, img_shape=(int(0),int(0))):
    if img_shape == (int(0),int(0)):
        _,_,w,h = cv2.boundingRect(ctr)
    else:
        h,w = img_shape
    img = np.zeros((h,w), 'uint8')
    img = cv2.drawContours(img, [ctr], -1, 255, -1)
    return img

# ctr: Should be output of create_contour_area_image() (Origin of points is the origin of bounding box)
def create_upright_solid_contour(ctr):
    ctr2 = ctr.copy()
    (cx,cy),(w,h),angle = cv2.minAreaRect(ctr2)
    M = cv2.getRotationMatrix2D((cx,cy), angle, 1)
    for i in range(ctr2.shape[0]):
        ctr2[i,0,:] = ( M @ np.array([ctr2[i,0,0], ctr2[i,0,1], 1]) ).astype('int')
    rect = cv2.boundingRect(ctr2)
    img = np.zeros((rect[3],rect[2]), 'uint8')
    ctr2 -= rect[0:2]
    M[:,2] -= rect[0:2]
    img = cv2.drawContours(img, [ctr2], -1, 255,-1)
    return img, M, ctr2


######################################################
# Dataset classes
######################################################
class contour_dataset:
    def __init__(self, ctr):
        self.ctr = ctr.copy()
        self.rrect = cv2.minAreaRect(ctr)
        self.box = cv2.boxPoints(self.rrect)
        self.solid = create_solid_contour(ctr)
        n = 100
        if n >= ctr.shape[0]:
            self.pts = np.array([p for p in ctr[:,0,:]])
        else:            
            r = n / ctr.shape[0]
            self.pts = np.zeros((100,2), 'int')
            pts = []
            for i in range(ctr.shape[0]):
                f = math.modf(i*r)[0] 
                if (f <= r/2) or (f > 1.0 - r/2):
                    pts += [ctr[i,0,:]]
            self.pts = np.array(pts)

class template_dataset:
    def __init__(self, ctr, num, selected_idx=[0]):
        self.ctr = ctr.copy()
        self.num = num
        self.rrect = cv2.minAreaRect(ctr)
        self.box = cv2.boxPoints(self.rrect)
        if num == 0:
            self.solid,_,_ = create_upright_solid_contour(ctr)
        else:
            self.solid = create_solid_contour(ctr)
        self.pts = np.array([ctr[idx,0,:] for idx in selected_idx])


######################################################
# ICP
######################################################
# pts: list of 2D points, or ndarray of shape (n,2)
# query: 2D point to find nearest neighbor
def find_nearest_neighbor(pts, query):
    min_distance_sq = float('inf')
    min_idx = 0
    for i, p in enumerate(pts):
        d = np.dot(query - p, query - p)
        if(d < min_distance_sq):
            min_distance_sq = d
            min_idx = i
    return min_idx, np.sqrt(min_distance_sq)

# src, dst: ndarray, shape is (n,2) (n: number of points)
def estimate_affine_2d(src, dst):
    n = min(src.shape[0], dst.shape[0])
    x = dst[0:n].flatten()
    A = np.zeros((2*n,6))
    for i in range(n):
        A[i*2,0] = src[i,0]
        A[i*2,1] = src[i,1]
        A[i*2,2] = 1
        A[i*2+1,3] = src[i,0]
        A[i*2+1,4] = src[i,1]
        A[i*2+1,5] = 1
    M = np.linalg.inv(A.T @ A) @ A.T @ x
    return M.reshape([2,3])

# Find optimum affine matrix using ICP algorithm
# src_pts: ndarray, shape is (n_s,2) (n_s: number of points)
# dst_pts: ndarray, shape is (n_d,2) (n_d: number of points, n_d should be larger or equal to n_s)
# initial_matrix: ndarray, shape is (2,3)
def icp(src_pts, dst_pts, max_iter=20, initial_matrix=np.array([[1.0, 0.0, 0.0], [0.0, 1.0, 0.0]])):
    default_affine_matrix = np.array([[1.0, 0.0, 0.0], [0.0, 1.0, 0.0]])
    if dst_pts.shape[0] < src_pts.shape[0]:
        # print("icp: Insufficient destination points")
        return default_affine_matrix, False
    if initial_matrix.shape != (2,3):
        print("icp: Illegal shape of initial_matrix")
        return default_affine_matrix, False
    M = initial_matrix
    # Store indices of the nearest neighbor point of dst_pts to the converted point of src_pts
    nn_idx = []
    for i in range(max_iter):
        nn_idx_tmp = []
        dst_pts_list = [p for p in dst_pts]
        idx_list = list(range(0,dst_pts.shape[0]))
        for p in src_pts:
            p2 = M @ np.array([p[0], p[1], 1])
            idx, d = find_nearest_neighbor(dst_pts_list, p2)
            nn_idx_tmp += [idx_list[idx]]
            del dst_pts_list[idx]
            del idx_list[idx]
        if nn_idx != [] and nn_idx == nn_idx_tmp:
            break
        dst_pts2 = np.zeros_like(src_pts)
        for j,idx in enumerate(nn_idx_tmp):
            dst_pts2[j,:] = dst_pts[idx,:]
        M = estimate_affine_2d(src_pts, dst_pts2)
        nn_idx = nn_idx_tmp
        if i == max_iter -1:
            return M, False
    return M, True


######################################################
# Calculating similarity and determining the number
######################################################
def binary_image_similarity(img1, img2):
    if img1.shape != img2.shape:
        print('binary_image_similarity: Different image size')
        return 0.0
    xor_img = cv2.bitwise_xor(img1, img2)
    return 1.0 - np.float(np.count_nonzero(xor_img)) / (img1.shape[0]*img2.shape[1])

# src, dst: contour_dataset or template_dataset (holding member variables box, solid)
def get_transform_by_rotated_rectangle(src, dst):
    # Rotated patterns are created when starting index is slided
    dst_box2 = np.vstack([dst.box, dst.box])
    max_similarity = 0.0
    max_converted_img = np.zeros((dst.solid.shape[1], dst.solid.shape[0]), 'uint8')
    for i in range(4):
        M = cv2.getAffineTransform(src.box[0:3], dst_box2[i:i+3])
        converted_img = cv2.warpAffine(src.solid, M, dsize=(dst.solid.shape[1], dst.solid.shape[0]), flags=cv2.INTER_NEAREST)
        similarity = binary_image_similarity(converted_img, dst.solid)
        if similarity > max_similarity:
            M_rtn = M
            max_similarity = similarity
            max_converted_img = converted_img
    return M_rtn, max_similarity, max_converted_img

def get_similarity_with_template(target_data, template_data, sim_th_high=0.95, sim_th_low=0.7):
    _,(w1,h1), _ = target_data.rrect
    _,(w2,h2), _ = template_data.rrect
    r = w1/h1 if w1 < h1 else h1/w1
    r = r * h2/w2 if w2 < h2 else r * w2/h2
    M, sim_init, _ = get_transform_by_rotated_rectangle(template_data, target_data)
    if sim_init > sim_th_high or sim_init < sim_th_low or r > 1.4 or r < 0.7:
        dsize = (template_data.solid.shape[1], template_data.solid.shape[0])
        flags = cv2.INTER_NEAREST|cv2.WARP_INVERSE_MAP
        converted_img = cv2.warpAffine(target_data.solid, M, dsize=dsize, flags=flags)
        return sim_init, converted_img
    M, _ = icp(template_data.pts, target_data.pts, initial_matrix=M)
    Minv = cv2.invertAffineTransform(M)
    converted_ctr = np.zeros_like(target_data.ctr)
    for i in range(target_data.ctr.shape[0]):
        converted_ctr[i,0,:] = (Minv[:,0:2] @ target_data.ctr[i,0,:]) + Minv[:,2]
    converted_img = create_solid_contour(converted_ctr, img_shape=template_data.solid.shape)
    val = binary_image_similarity(converted_img, template_data.solid)
    return val, converted_img

def get_similarity_with_template_zero(target_data, template_data):
    dsize = (template_data.solid.shape[1], template_data.solid.shape[0])
    converted_img = cv2.resize(target_data.solid, dsize=dsize, interpolation=cv2.INTER_NEAREST)
    val = binary_image_similarity(converted_img, template_data.solid)
    return val, converted_img

def get_similarities(target, templates):
    similarities = []
    converted_imgs = []
    for tmpl in templates:
        if tmpl.num == 0:
            sim,converted_img = get_similarity_with_template_zero(target, tmpl)
        else:
            sim,converted_img = get_similarity_with_template(target, tmpl)
        similarities += [sim]
        converted_imgs += [converted_img]
    return similarities, converted_imgs

def calc_harupan(img, templates, svm):
    ctrs, resized_img = detect_candidate_contours(img, sat_th=50)
    # print('Number of candidates: ', len(ctrs))
    if len(ctrs) == 0:
        return 0.0, resized_img
    subctrs, _, _ = refine_contours(resized_img, ctrs)
    subctr_datasets = [contour_dataset(ctr) for ctr in subctrs]
    ########
    #### Simple code
    # similarities = [get_similarities(d, templates)[0] for d in subctr_datasets]
    #### Code printing progress
    similarities = []
    for i,d in enumerate(subctr_datasets):
        print(i, end=' ')
        similarities += [get_similarities(d, templates)[0]]
        print(similarities[-1])
        print('')
    print('')
    #######
    _, result = svm.predict(np.array(similarities, 'float32'))
    result = result.astype('int')
    score = 0.0
    texts = {0:'0', 1:'1', 2:'2', 3:'3', 5:'.5'}
    font = cv2.FONT_HERSHEY_SIMPLEX
    for res, ctr in zip(result, ctrs):
        if res[0] == 5:
            score += 0.5
        elif res[0] != -1:
            score += res[0]
        
        # Annotating recognized numbers for confirmation
        if res[0] != -1:
            resized_img = cv2.drawContours(resized_img, [ctr], -1, (0,255,0), 3)
            x,y,_,_ = cv2.boundingRect(ctr)
            resized_img = cv2.putText(resized_img, texts[res[0]], (x,y), font, 1, (230,230,0), 5)
    return score, resized_img

######################################################
# Loading template data and SVM model
######################################################
def load_svm(filename):
    return cv2.ml.SVM_load(filename)

def load_templates(filename):
    with open(filename, mode='r') as f:
        load_data = json.load(f)
        templates_rtn = []
        for d in load_data:
            templates_rtn += [template_dataset(np.array(d['ctr']), d['num'], d['pts'])]
    return templates_rtn

テンプレートデータ、SVMデータ読み込み

svm = load_svm('harupan_data/harupan_svm_220412.dat')
templates2019 = load_templates('harupan_data/templates2019.json')
templates2020 = load_templates('harupan_data/templates2020.json')
templates2021 = load_templates('harupan_data/templates2021.json')

ついでに。

%matplotlib inline

処理変更

今回のICP処理では、テンプレートの輪郭点(ある程度間引いているので、点数少なめ)について、画像上で検出した輪郭上の輪郭点の中から最近傍の点を探して、それをマッチング点としてアフィン変換行列を推定しています。

この最近傍点探索処理を、以下のように変更します。

• 最初の1点は、対象輪郭の全点から最近傍点を探す
• 2点目以降は、前のマッチング点周辺の所定の範囲の点から最近傍点を探す

ICP処理関数の引数として、探索範囲を示すsearch_rangeを追加します。
まずは0.5に設定して、対象輪郭点のうち半分からの探索になるようにします。
ICP処理時間が半分になることを期待。

この引数を変えて試してみることを考えると、ICP関数は2段構えにしておいたほうがいいかも。

まずはデバッグ表示を色々付けた状態で。

def create_pts_image(pts):
    ctr = np.zeros((len(pts), 1, 2), 'int')
    for i,p in enumerate(pts):
        ctr[i,0,:] = p
    x,y,w,h = cv2.boundingRect(ctr)
    xe = x+w
    ye = y+h
    img = np.zeros((ye,xe,3), 'uint8')
    for i,p in enumerate(pts):
        cv2.drawMarker(img, p, (0,255,0), markerType=cv2.MARKER_CROSS, markerSize=3)
    cv2.drawMarker(img, pts[0], (255,0,0), markerType=cv2.MARKER_CROSS, markerSize=3)
    return img

# Find optimum affine matrix using ICP algorithm
# src_pts: ndarray, shape is (n_s,2) (n_s: number of points)
# dst_pts: ndarray, shape is (n_d,2) (n_d: number of points, n_d should be larger or equal to n_s)
# initial_matrix: ndarray, shape is (2,3)
def icp(src_pts, dst_pts, max_iter=20, initial_matrix=np.array([[1.0, 0.0, 0.0], [0.0, 1.0, 0.0]])):
    search_range = 0.5
    return icp_sub(src_pts, dst_pts, max_iter=max_iter, initial_matrix=initial_matrix, search_range=search_range)

# Find optimum affine matrix using ICP algorithm
# src_pts: ndarray, shape is (n_s,2) (n_s: number of points)
# dst_pts: ndarray, shape is (n_d,2) (n_d: number of points, n_d should be larger or equal to n_s)
# initial_matrix: ndarray, shape is (2,3)
# search_range: float number, 0.0 ~ 1.0, the range to search nearest neighbor, 1.0 -> Search in all dst_pts
def icp_sub(src_pts, dst_pts, max_iter=20, initial_matrix=np.array([[1.0, 0.0, 0.0], [0.0, 1.0, 0.0]]), search_range=0.5):
    default_affine_matrix = np.array([[1.0, 0.0, 0.0], [0.0, 1.0, 0.0]])
    n_dst = dst_pts.shape[0]
    n_src = src_pts.shape[0]
    if n_dst < n_src:
        # print("icp: Insufficient destination points")
        return default_affine_matrix, False
    if initial_matrix.shape != (2,3):
        print("icp: Illegal shape of initial_matrix")
        return default_affine_matrix, False
    n_search = int(n_dst*search_range)
    M = initial_matrix
    # Store indices of the nearest neighbor point of dst_pts to the converted point of src_pts
    nn_idx = []
    converged = False
    for i in range(max_iter):
        nn_idx_tmp = []
        dst_pts_list = [p for p in dst_pts]
        idx_list = list(range(0,dst_pts.shape[0]))
        first_pt = True
        for p in src_pts:
            # Convert source point with current conversion matrix
            p2 = M @ np.array([p[0], p[1], 1])
            if first_pt:
                # First point should be searched in all destination points
                idx, _ = find_nearest_neighbor(dst_pts_list, p2)
                first_pt = False
            else:
                # Search nearest neighbor point in specified range around the last point
                n = int(min(n_search/2, len(idx_list)/2))
                s = max(len(idx_list) + last_idx - n, 0)
                e = min(len(idx_list) + last_idx + n, 3*len(idx_list))
                pts = (dst_pts_list + dst_pts_list + dst_pts_list)[s:e]
                idx, _ = find_nearest_neighbor(pts, p2)
                # The index acquired above is counted from 's', so actual index must be recovered
                idx = (idx + s) % len(idx_list)
            nn_idx_tmp += [idx_list[idx]]
            last_idx = idx
            del dst_pts_list[idx]
            del idx_list[idx]
        print('nn_idx: ', nn_idx_tmp)
        if nn_idx != [] and nn_idx == nn_idx_tmp:
            converged = True
            break
        dst_pts2 = np.zeros_like(src_pts)
        for j,idx in enumerate(nn_idx_tmp):
            dst_pts2[j,:] = dst_pts[idx,:]
        M = estimate_affine_2d(src_pts, dst_pts2)
        nn_idx = nn_idx_tmp
    #### Debug ####
    pts1 = []
    pts2 = []
    for p in src_pts:
        p1 = initial_matrix @ np.array([p[0], p[1], 1])
        pts1 += [p1.astype('int')]
        p2 = M @ np.array([p[0], p[1], 1])
        pts2 += [p2.astype('int')]
    plt.subplot(1,3,1), plt.imshow(create_pts_image(pts1)), plt.xticks([]), plt.yticks([])
    plt.subplot(1,3,2), plt.imshow(create_pts_image(dst_pts)), plt.xticks([]), plt.yticks([])
    plt.subplot(1,3,3), plt.imshow(create_pts_image(pts2)), plt.xticks([]), plt.yticks([])
    plt.show()
    ###############
    return M, converged

変更処理試し

今まで使っていた画像の1つで試してみます。

ちなみに、上記の処理に追加しているデバッグ処理では、

• 最近傍点リストの全経過を表示
• 3つの輪郭点画像を表示
  • テンプレート輪郭を初期変換行列で変換したもの
  • 対象輪郭点
  • テンプレート輪郭を最終の変換行列で変換したもの

ということをやっています。

img1 = cv2.imread('harupan_190428_1.jpg')

img = img1
templates = templates2019

# test_harupan_timeit(img, templates, svm)

score, result_img = calc_harupan(img, templates, svm)
print('Score: ', score)
plt.figure(figsize=(6.4,4.8), dpi=200)
plt.imshow(cv2.cvtColor(result_img, cv2.COLOR_BGR2RGB)), plt.xticks([]), plt.yticks([])
plt.show()

0 nn_idx:  [104, 102, 99, 98, 95, 91, 90, 88, 67, 65, 64, 62, 53, 58, 59, 60, 57, 55, 54, 51, 49, 47, 46, 44, 42, 39, 36, 29, 28, 14, 15, 16, 87, 83, 81, 78, 77, 5, 6, 7, 3, 0, 105]
nn_idx:  [103, 102, 99, 97, 95, 92, 90, 89, 67, 65, 64, 62, 53, 58, 59, 60, 57, 55, 54, 52, 50, 48, 46, 44, 42, 39, 40, 29, 26, 14, 15, 16, 87, 83, 82, 78, 77, 5, 6, 4, 3, 0, 105]
nn_idx:  [103, 101, 99, 97, 95, 92, 90, 89, 67, 65, 64, 62, 53, 58, 59, 60, 57, 55, 54, 52, 50, 48, 46, 45, 42, 40, 41, 28, 26, 14, 15, 16, 87, 84, 82, 78, 77, 5, 6, 4, 3, 0, 104]
nn_idx:  [103, 101, 99, 97, 95, 92, 90, 89, 67, 65, 64, 62, 53, 58, 59, 60, 57, 55, 54, 52, 50, 48, 47, 45, 43, 41, 40, 28, 26, 16, 15, 17, 88, 84, 82, 78, 77, 5, 6, 4, 3, 0, 104]
nn_idx:  [103, 101, 99, 97, 95, 92, 90, 89, 67, 65, 64, 62, 53, 58, 59, 60, 57, 56, 54, 52, 50, 49, 47, 45, 43, 41, 40, 28, 25, 16, 15, 17, 88, 84, 82, 78, 77, 5, 6, 4, 3, 0, 104]
nn_idx:  [103, 101, 99, 97, 95, 92, 90, 89, 67, 65, 64, 62, 53, 58, 59, 60, 57, 56, 54, 52, 50, 49, 47, 45, 43, 41, 40, 27, 25, 16, 15, 17, 88, 84, 82, 78, 77, 5, 6, 4, 3, 0, 104]
nn_idx:  [103, 101, 99, 97, 95, 92, 90, 89, 67, 65, 64, 62, 53, 58, 59, 60, 57, 56, 54, 52, 50, 49, 47, 45, 43, 41, 40, 27, 25, 16, 15, 17, 88, 84, 82, 78, 77, 5, 6, 4, 3, 0, 104]

nn_idx:  [102, 100, 98, 96, 92, 90, 89, 68, 67, 65, 62, 60, 58, 55, 51, 49, 48, 46, 43, 40, 35, 33, 32, 31, 29, 26, 25, 23, 20, 18, 15, 14, 13, 16, 87, 83, 80, 78, 4, 5, 3, 1, 103]
nn_idx:  [102, 100, 98, 96, 92, 90, 89, 68, 67, 65, 62, 60, 58, 55, 51, 49, 47, 45, 44, 41, 36, 34, 32, 31, 29, 26, 25, 23, 21, 18, 16, 14, 13, 15, 87, 83, 79, 78, 4, 5, 3, 1, 105]
nn_idx:  [102, 100, 98, 96, 92, 90, 89, 68, 67, 65, 62, 60, 58, 54, 51, 49, 47, 45, 44, 41, 36, 34, 32, 31, 29, 26, 25, 23, 21, 19, 16, 14, 13, 15, 87, 83, 79, 78, 5, 6, 3, 1, 105]
nn_idx:  [102, 100, 98, 96, 92, 90, 89, 68, 67, 65, 62, 60, 58, 54, 51, 49, 47, 45, 44, 41, 36, 34, 33, 32, 29, 26, 25, 24, 21, 19, 16, 14, 13, 15, 87, 83, 79, 78, 5, 6, 4, 1, 105]
nn_idx:  [102, 100, 98, 96, 92, 90, 89, 68, 67, 65, 62, 60, 58, 54, 51, 49, 47, 45, 44, 41, 36, 34, 33, 32, 29, 26, 25, 24, 21, 19, 16, 14, 13, 15, 87, 83, 79, 78, 5, 6, 4, 1, 105]

nn_idx:  [0, 105, 101, 97, 94, 85, 83, 78, 74, 69, 65, 61, 56, 51, 47, 44, 39, 35, 31, 26, 23, 16, 12, 9, 8, 6, 3]
nn_idx:  [0, 104, 101, 96, 92, 86, 83, 78, 74, 69, 65, 61, 56, 50, 47, 44, 41, 35, 30, 26, 23, 16, 12, 9, 8, 5, 1]
nn_idx:  [0, 103, 100, 96, 92, 86, 83, 78, 74, 69, 65, 61, 55, 50, 47, 44, 41, 35, 30, 26, 23, 17, 14, 9, 8, 5, 1]
nn_idx:  [0, 103, 99, 96, 91, 86, 83, 77, 74, 69, 65, 61, 55, 50, 47, 44, 41, 35, 30, 26, 23, 17, 14, 10, 8, 4, 1]
nn_idx:  [106, 103, 99, 96, 91, 86, 83, 77, 73, 69, 65, 61, 55, 50, 47, 44, 41, 35, 29, 25, 23, 17, 14, 10, 8, 4, 1]
nn_idx:  [106, 103, 99, 96, 91, 86, 83, 77, 73, 69, 65, 61, 55, 50, 47, 44, 41, 35, 29, 25, 23, 17, 14, 10, 8, 4, 1]

[0.5932692307692308, 0.762987012987013, 0.6706689536878216, 0.7790697674418605, 0.8884848484848484]

1 nn_idx:  [98, 97, 94, 92, 90, 86, 84, 66, 65, 63, 61, 58, 51, 54, 56, 57, 55, 53, 52, 50, 49, 48, 45, 44, 42, 39, 40, 31, 29, 18, 19, 17, 20, 67, 77, 74, 73, 6, 8, 7, 5, 2, 99]
nn_idx:  [98, 96, 94, 92, 90, 85, 84, 66, 65, 63, 61, 58, 51, 54, 56, 57, 55, 53, 52, 50, 49, 48, 45, 44, 42, 41, 40, 31, 29, 19, 18, 17, 20, 67, 77, 74, 73, 6, 8, 7, 5, 2, 99]
nn_idx:  [98, 96, 94, 92, 90, 85, 84, 66, 65, 63, 61, 58, 51, 54, 56, 57, 55, 53, 52, 50, 49, 48, 46, 44, 42, 41, 40, 31, 29, 19, 18, 20, 67, 70, 77, 73, 72, 6, 8, 7, 5, 2, 99]
nn_idx:  [98, 96, 94, 92, 90, 85, 84, 66, 65, 63, 61, 58, 51, 54, 56, 57, 55, 53, 52, 50, 49, 48, 46, 44, 42, 41, 40, 31, 29, 19, 18, 20, 67, 70, 77, 73, 72, 6, 8, 7, 5, 2, 99]

nn_idx:  [97, 94, 92, 90, 87, 84, 85, 66, 65, 63, 60, 58, 56, 55, 52, 50, 48, 45, 44, 41, 36, 34, 35, 32, 31, 29, 27, 24, 23, 21, 19, 18, 17, 67, 80, 79, 76, 74, 6, 7, 4, 2, 98]
nn_idx:  [97, 94, 92, 90, 87, 85, 84, 66, 65, 63, 61, 59, 56, 55, 52, 50, 48, 45, 44, 41, 38, 35, 34, 32, 31, 29, 27, 25, 23, 21, 19, 18, 17, 67, 79, 78, 76, 74, 6, 7, 5, 2, 98]
nn_idx:  [97, 94, 92, 90, 87, 85, 84, 66, 65, 63, 61, 59, 56, 55, 52, 50, 48, 45, 44, 41, 38, 36, 34, 32, 31, 29, 27, 25, 23, 21, 19, 18, 17, 67, 79, 78, 75, 74, 6, 7, 5, 2, 99]
nn_idx:  [97, 95, 92, 90, 87, 85, 84, 66, 65, 63, 61, 59, 56, 55, 52, 50, 48, 45, 44, 41, 38, 36, 34, 32, 31, 29, 27, 25, 23, 21, 19, 18, 17, 67, 79, 78, 75, 74, 6, 7, 5, 3, 99]
nn_idx:  [97, 95, 92, 91, 87, 85, 84, 66, 65, 63, 61, 59, 57, 55, 52, 50, 48, 45, 44, 41, 38, 36, 34, 32, 31, 29, 27, 25, 23, 21, 19, 18, 17, 67, 70, 78, 75, 74, 6, 7, 5, 3, 99]
nn_idx:  [97, 95, 93, 91, 87, 85, 84, 66, 65, 63, 61, 59, 57, 55, 52, 50, 48, 45, 44, 41, 38, 36, 34, 32, 31, 29, 27, 25, 23, 21, 19, 18, 17, 67, 70, 78, 75, 74, 6, 7, 5, 3, 99]
nn_idx:  [97, 95, 93, 91, 87, 85, 84, 66, 65, 63, 61, 59, 57, 55, 52, 50, 48, 45, 44, 41, 38, 36, 34, 32, 31, 29, 27, 25, 23, 21, 19, 18, 17, 67, 70, 78, 75, 74, 6, 7, 5, 3, 99]

nn_idx:  [1, 99, 95, 91, 87, 81, 77, 74, 70, 66, 63, 59, 55, 51, 47, 44, 41, 37, 33, 29, 25, 19, 16, 11, 9, 7, 3]
nn_idx:  [1, 99, 95, 91, 86, 81, 79, 73, 70, 66, 63, 59, 55, 51, 47, 44, 41, 37, 32, 29, 25, 19, 16, 11, 9, 7, 3]
nn_idx:  [1, 98, 95, 91, 86, 81, 79, 73, 70, 66, 63, 59, 55, 51, 47, 44, 41, 37, 32, 29, 25, 19, 16, 11, 9, 6, 3]
nn_idx:  [1, 98, 94, 91, 86, 82, 79, 73, 69, 66, 63, 59, 55, 51, 47, 44, 41, 37, 32, 29, 25, 19, 16, 11, 9, 6, 3]
nn_idx:  [1, 98, 94, 90, 86, 82, 79, 73, 69, 66, 63, 59, 55, 51, 47, 44, 41, 37, 32, 29, 25, 19, 16, 11, 9, 6, 3]
nn_idx:  [1, 98, 94, 90, 86, 82, 79, 73, 69, 66, 63, 59, 55, 51, 47, 44, 41, 37, 32, 29, 25, 19, 16, 11, 9, 6, 3]

[0.5177884615384616, 0.7431761786600497, 0.6312178387650086, 0.7769933554817275, 0.8715151515151516]

2 nn_idx:  [1, 99, 96, 93, 90, 86, 82, 78, 75, 71, 67, 64, 60, 55, 52, 49, 45, 42, 38, 35, 30, 27, 23, 19, 15, 10, 7, 5, 2]
nn_idx:  [1, 99, 96, 93, 90, 86, 82, 78, 75, 71, 67, 64, 60, 55, 52, 48, 45, 42, 38, 35, 30, 27, 23, 19, 15, 10, 8, 5, 2]
nn_idx:  [1, 99, 96, 93, 90, 86, 82, 78, 75, 71, 67, 64, 60, 55, 52, 48, 45, 42, 38, 35, 30, 27, 23, 19, 15, 10, 8, 5, 2]

[0.7173076923076923, 0.927257525083612, 0.7626208378088077, 0.7422126745435016, 0.7035445757250269]

3 nn_idx:  [95, 94, 91, 90, 87, 86, 84, 66, 64, 63, 61, 59, 53, 54, 56, 57, 55, 52, 51, 50, 48, 47, 45, 43, 41, 39, 38, 32, 29, 17, 19, 18, 65, 81, 79, 72, 73, 74, 6, 5, 3, 99, 96]
nn_idx:  [96, 94, 92, 90, 87, 86, 84, 66, 64, 63, 61, 59, 53, 54, 56, 57, 55, 52, 51, 50, 48, 47, 44, 43, 41, 40, 38, 31, 29, 17, 19, 18, 65, 81, 79, 72, 73, 74, 6, 5, 3, 99, 97]
nn_idx:  [96, 94, 92, 91, 87, 86, 84, 66, 64, 63, 61, 59, 53, 54, 56, 57, 55, 52, 51, 50, 49, 47, 45, 43, 41, 40, 38, 31, 29, 17, 19, 18, 65, 81, 79, 72, 73, 74, 6, 5, 3, 99, 97]
nn_idx:  [96, 94, 92, 91, 87, 86, 84, 66, 64, 63, 61, 59, 53, 54, 56, 57, 55, 52, 51, 50, 49, 47, 45, 43, 42, 40, 38, 31, 29, 17, 19, 18, 65, 81, 79, 72, 73, 74, 6, 5, 3, 99, 97]
nn_idx:  [96, 94, 92, 91, 87, 86, 84, 66, 64, 63, 61, 59, 53, 54, 56, 57, 55, 52, 51, 50, 49, 47, 45, 43, 42, 40, 32, 31, 29, 17, 19, 18, 65, 81, 79, 72, 73, 74, 6, 5, 3, 99, 97]
nn_idx:  [96, 94, 92, 91, 87, 86, 84, 66, 64, 63, 61, 59, 53, 54, 56, 57, 55, 52, 51, 50, 49, 47, 46, 44, 42, 40, 31, 32, 29, 19, 18, 20, 67, 81, 79, 72, 73, 74, 6, 5, 3, 99, 97]
nn_idx:  [96, 94, 92, 91, 87, 86, 84, 66, 64, 63, 61, 59, 53, 54, 56, 57, 55, 52, 51, 50, 49, 48, 46, 44, 42, 40, 31, 30, 29, 19, 18, 20, 67, 81, 79, 72, 73, 74, 6, 5, 3, 99, 97]
nn_idx:  [96, 94, 92, 91, 87, 86, 84, 66, 64, 63, 61, 59, 53, 54, 56, 57, 55, 52, 51, 50, 49, 48, 46, 44, 43, 40, 31, 30, 29, 19, 18, 20, 67, 81, 79, 72, 73, 74, 6, 5, 3, 99, 97]
nn_idx:  [96, 94, 92, 91, 87, 86, 84, 66, 64, 63, 61, 59, 53, 54, 56, 57, 55, 52, 51, 50, 49, 48, 46, 44, 43, 40, 31, 30, 29, 19, 20, 18, 67, 81, 79, 72, 73, 74, 6, 5, 3, 99, 97]
nn_idx:  [96, 94, 92, 91, 87, 86, 84, 66, 64, 63, 61, 59, 53, 54, 56, 57, 55, 52, 51, 50, 49, 48, 46, 44, 43, 40, 31, 30, 29, 19, 20, 18, 67, 81, 79, 72, 73, 74, 6, 5, 3, 99, 97]

nn_idx:  [94, 92, 90, 88, 86, 84, 66, 64, 65, 62, 59, 57, 55, 54, 51, 49, 47, 45, 43, 40, 36, 34, 11, 12, 15, 29, 27, 25, 23, 22, 19, 18, 17, 20, 85, 80, 78, 72, 75, 5, 3, 99, 96]
nn_idx:  [95, 92, 91, 87, 86, 84, 66, 64, 65, 62, 59, 57, 55, 52, 51, 48, 47, 45, 43, 40, 33, 34, 11, 12, 15, 29, 27, 25, 23, 21, 19, 18, 17, 20, 85, 80, 78, 74, 75, 5, 2, 99, 96]
nn_idx:  [95, 92, 91, 89, 86, 84, 66, 65, 64, 61, 59, 56, 55, 52, 51, 48, 47, 45, 43, 31, 33, 34, 11, 12, 15, 29, 27, 25, 23, 21, 19, 18, 17, 20, 85, 79, 78, 75, 74, 5, 2, 99, 96]
nn_idx:  [95, 93, 91, 89, 85, 84, 66, 65, 64, 61, 59, 56, 55, 52, 51, 49, 47, 45, 44, 31, 33, 34, 11, 12, 15, 29, 27, 25, 23, 21, 19, 18, 17, 20, 86, 79, 78, 75, 74, 3, 2, 99, 96]
nn_idx:  [95, 93, 91, 90, 85, 84, 66, 65, 64, 61, 59, 56, 55, 52, 51, 49, 47, 45, 44, 31, 33, 34, 11, 12, 15, 29, 27, 25, 23, 21, 19, 18, 17, 20, 86, 79, 78, 75, 74, 3, 2, 99, 96]
nn_idx:  [95, 93, 91, 90, 85, 84, 66, 65, 64, 61, 59, 56, 55, 52, 51, 49, 47, 45, 44, 31, 33, 34, 11, 12, 15, 29, 27, 25, 23, 21, 19, 18, 17, 20, 86, 79, 78, 75, 74, 3, 2, 99, 96]

nn_idx:  [100, 97, 94, 89, 86, 83, 79, 72, 69, 66, 62, 58, 55, 51, 47, 43, 40, 36, 33, 29, 25, 19, 15, 12, 8, 4, 1]
nn_idx:  [100, 97, 94, 89, 86, 83, 79, 72, 70, 66, 62, 58, 55, 51, 47, 43, 40, 36, 33, 29, 25, 19, 15, 12, 8, 5, 1]
nn_idx:  [100, 97, 94, 89, 86, 83, 79, 72, 70, 66, 62, 58, 55, 50, 47, 43, 40, 36, 33, 29, 25, 19, 15, 12, 8, 5, 1]
nn_idx:  [100, 97, 94, 89, 86, 83, 79, 72, 70, 66, 62, 58, 55, 50, 47, 43, 40, 36, 33, 29, 25, 19, 15, 12, 8, 5, 1]

[0.5129807692307693, 0.7327188940092166, 0.6183533447684391, 0.6362126245847176, 0.8739393939393939]

4 nn_idx:  [4, 2, 0, 97, 95, 92, 91, 89, 86, 84, 81, 79, 77, 73, 71, 69, 67, 65, 62, 60, 57, 55, 53, 51, 49, 47, 44, 41, 39, 37, 34, 31, 29, 26, 24, 21, 20, 18, 15, 12, 10, 7, 5]
nn_idx:  [4, 2, 0, 98, 95, 93, 91, 88, 86, 84, 81, 79, 77, 73, 71, 69, 67, 65, 62, 60, 58, 55, 53, 51, 49, 47, 44, 41, 39, 37, 34, 31, 29, 26, 24, 21, 19, 17, 15, 12, 10, 7, 5]
nn_idx:  [4, 2, 0, 98, 95, 93, 91, 88, 86, 84, 81, 79, 77, 73, 71, 69, 67, 65, 62, 60, 58, 56, 53, 51, 49, 47, 44, 41, 39, 37, 34, 31, 29, 26, 24, 21, 19, 17, 15, 12, 10, 7, 5]
nn_idx:  [4, 2, 0, 98, 95, 93, 91, 88, 86, 84, 81, 79, 77, 73, 71, 69, 67, 65, 62, 60, 58, 56, 53, 51, 49, 47, 44, 41, 39, 37, 34, 31, 29, 26, 24, 21, 19, 17, 15, 12, 10, 7, 5]

nn_idx:  [2, 0, 98, 95, 93, 91, 89, 87, 85, 83, 82, 69, 68, 66, 60, 58, 56, 53, 51, 43, 44, 42, 41, 39, 38, 37, 76, 75, 78, 79, 36, 34, 31, 29, 27, 25, 23, 21, 17, 15, 9, 7, 4]
nn_idx:  [2, 0, 98, 95, 93, 91, 89, 87, 85, 83, 82, 69, 68, 66, 61, 59, 56, 55, 52, 50, 43, 42, 41, 39, 38, 37, 76, 75, 78, 79, 36, 34, 31, 30, 27, 26, 23, 20, 18, 16, 9, 7, 4]
nn_idx:  [2, 0, 98, 96, 93, 91, 89, 87, 85, 83, 82, 69, 68, 66, 61, 59, 57, 55, 53, 51, 43, 42, 41, 39, 38, 37, 76, 75, 78, 79, 36, 34, 31, 30, 27, 26, 23, 20, 19, 16, 9, 7, 4]
nn_idx:  [2, 0, 98, 96, 94, 91, 89, 87, 85, 83, 82, 69, 68, 66, 61, 59, 58, 56, 53, 51, 43, 42, 41, 39, 38, 37, 76, 75, 78, 79, 36, 34, 31, 30, 27, 26, 23, 20, 19, 17, 9, 7, 4]
nn_idx:  [2, 0, 98, 96, 94, 92, 90, 87, 85, 83, 82, 69, 68, 66, 62, 59, 58, 56, 53, 51, 43, 41, 40, 39, 38, 37, 76, 75, 78, 79, 36, 34, 31, 30, 27, 26, 23, 20, 19, 17, 9, 7, 4]
nn_idx:  [2, 0, 98, 96, 94, 92, 90, 87, 85, 83, 82, 69, 68, 66, 62, 60, 58, 56, 53, 51, 42, 41, 40, 39, 38, 37, 76, 75, 78, 79, 36, 34, 31, 30, 27, 26, 23, 20, 19, 17, 9, 7, 4]
nn_idx:  [2, 0, 98, 96, 94, 92, 90, 87, 85, 83, 82, 69, 68, 66, 62, 60, 58, 56, 54, 52, 42, 41, 40, 39, 38, 37, 76, 75, 78, 79, 36, 34, 31, 30, 27, 26, 23, 20, 19, 17, 9, 7, 4]
nn_idx:  [2, 0, 98, 96, 94, 92, 90, 87, 85, 83, 82, 69, 68, 66, 62, 60, 58, 56, 54, 52, 42, 41, 40, 39, 38, 37, 76, 75, 78, 79, 36, 34, 31, 30, 27, 26, 23, 20, 19, 17, 9, 7, 4]

[0.60625, 0.7112299465240641, 0.9240994854202401, 0.7333887043189369, 0.6907308377896613]

5 nn_idx:  [1, 97, 93, 90, 86, 83, 80, 76, 73, 69, 66, 62, 58, 54, 51, 50, 46, 43, 37, 34, 30, 27, 23, 20, 16, 14, 8, 5, 2]
nn_idx:  [1, 97, 93, 90, 87, 83, 80, 76, 73, 69, 65, 62, 58, 54, 51, 48, 45, 42, 37, 34, 30, 27, 23, 20, 16, 11, 8, 5, 2]
nn_idx:  [1, 97, 93, 90, 86, 83, 80, 76, 73, 69, 65, 62, 58, 54, 51, 48, 45, 41, 37, 33, 30, 27, 23, 20, 16, 11, 8, 5, 2]
nn_idx:  [1, 97, 93, 90, 86, 83, 80, 76, 73, 69, 65, 62, 58, 54, 51, 48, 44, 41, 37, 33, 30, 26, 23, 20, 16, 11, 8, 5, 2]
nn_idx:  [1, 97, 93, 90, 86, 83, 80, 76, 73, 69, 65, 62, 58, 54, 51, 48, 44, 41, 37, 33, 30, 26, 23, 19, 16, 11, 8, 5, 2]
nn_idx:  [1, 97, 93, 90, 86, 83, 80, 76, 73, 69, 65, 62, 58, 54, 51, 48, 44, 41, 37, 33, 30, 26, 23, 19, 16, 11, 8, 4, 2]
nn_idx:  [1, 97, 93, 90, 86, 83, 80, 76, 73, 69, 65, 62, 58, 54, 51, 48, 44, 41, 37, 33, 30, 26, 23, 19, 16, 11, 8, 4, 2]

[0.66875, 0.9297658862876255, 0.7671614100185529, 0.7374768089053803, 0.7476808905380334]

6 nn_idx:  [4, 2, 0, 98, 95, 93, 91, 89, 87, 85, 82, 80, 77, 74, 72, 70, 68, 65, 63, 61, 59, 56, 54, 52, 49, 47, 44, 42, 39, 37, 33, 31, 28, 26, 24, 21, 19, 17, 15, 12, 10, 7, 5]
nn_idx:  [4, 2, 0, 98, 95, 93, 91, 89, 87, 85, 82, 80, 77, 74, 72, 70, 67, 65, 63, 61, 59, 56, 54, 52, 49, 47, 44, 42, 39, 37, 33, 31, 28, 26, 24, 21, 19, 17, 15, 12, 10, 7, 5]
nn_idx:  [4, 2, 0, 98, 95, 93, 91, 89, 87, 85, 81, 79, 77, 74, 72, 70, 67, 65, 63, 61, 59, 56, 54, 52, 49, 47, 44, 42, 39, 37, 33, 31, 28, 26, 24, 21, 19, 17, 15, 12, 10, 7, 5]
nn_idx:  [4, 2, 0, 98, 95, 93, 91, 89, 87, 84, 81, 79, 77, 74, 72, 70, 67, 65, 63, 61, 58, 56, 54, 52, 49, 47, 44, 42, 39, 37, 33, 31, 28, 26, 24, 21, 19, 17, 15, 12, 10, 7, 5]
nn_idx:  [4, 2, 0, 98, 95, 93, 91, 89, 87, 84, 81, 79, 77, 74, 72, 70, 67, 65, 63, 61, 58, 56, 54, 52, 49, 47, 44, 42, 39, 37, 33, 31, 28, 26, 24, 21, 19, 17, 15, 12, 10, 7, 5]

nn_idx:  [2, 0, 98, 96, 94, 92, 90, 88, 85, 84, 83, 70, 69, 66, 61, 59, 57, 55, 52, 50, 43, 42, 41, 40, 38, 39, 37, 77, 78, 79, 36, 33, 32, 29, 27, 25, 23, 20, 18, 15, 9, 7, 4]
nn_idx:  [2, 0, 98, 96, 94, 92, 90, 88, 85, 84, 83, 70, 69, 66, 62, 60, 58, 55, 53, 51, 43, 42, 41, 40, 38, 39, 76, 77, 78, 79, 36, 33, 32, 29, 27, 25, 23, 20, 18, 16, 9, 7, 4]
nn_idx:  [2, 0, 98, 96, 94, 92, 90, 88, 85, 84, 83, 69, 70, 66, 62, 60, 58, 56, 54, 52, 43, 42, 41, 40, 38, 37, 76, 75, 78, 79, 36, 33, 32, 29, 27, 25, 23, 20, 18, 16, 9, 7, 4]
nn_idx:  [2, 1, 98, 96, 94, 92, 90, 88, 85, 84, 83, 69, 68, 66, 62, 60, 58, 56, 55, 52, 43, 42, 41, 39, 38, 37, 76, 75, 78, 79, 36, 33, 32, 29, 27, 25, 23, 20, 18, 16, 9, 7, 4]
nn_idx:  [3, 1, 99, 96, 94, 92, 90, 88, 85, 84, 83, 69, 68, 66, 62, 60, 58, 56, 55, 52, 43, 42, 41, 39, 38, 37, 76, 75, 78, 79, 36, 33, 32, 29, 27, 25, 23, 20, 18, 16, 9, 7, 4]
nn_idx:  [3, 1, 99, 97, 94, 92, 90, 88, 85, 84, 83, 69, 68, 66, 63, 61, 58, 56, 55, 52, 43, 42, 41, 39, 38, 37, 76, 75, 78, 79, 36, 33, 32, 29, 27, 25, 23, 20, 18, 16, 9, 7, 4]
nn_idx:  [3, 1, 99, 97, 94, 92, 90, 88, 85, 84, 83, 69, 68, 66, 63, 61, 59, 57, 55, 52, 43, 42, 41, 39, 38, 37, 76, 75, 78, 79, 36, 33, 32, 29, 27, 25, 23, 20, 18, 16, 9, 7, 4]
nn_idx:  [3, 1, 99, 97, 94, 92, 90, 88, 85, 84, 83, 69, 68, 66, 63, 61, 59, 57, 55, 53, 42, 41, 40, 39, 38, 37, 76, 75, 78, 79, 36, 33, 32, 29, 27, 25, 23, 20, 18, 16, 9, 7, 5]
nn_idx:  [3, 1, 99, 97, 94, 92, 90, 87, 85, 84, 83, 69, 68, 66, 63, 61, 59, 58, 55, 53, 42, 41, 40, 39, 38, 37, 76, 75, 78, 79, 36, 33, 32, 29, 27, 25, 23, 20, 18, 16, 9, 7, 5]
nn_idx:  [3, 1, 99, 97, 94, 92, 90, 87, 85, 84, 83, 69, 68, 66, 63, 61, 60, 58, 55, 53, 42, 41, 40, 39, 38, 37, 76, 75, 78, 79, 36, 33, 32, 29, 27, 25, 23, 20, 18, 16, 9, 7, 5]
nn_idx:  [3, 1, 99, 97, 94, 92, 90, 87, 85, 84, 83, 69, 68, 66, 63, 61, 60, 58, 55, 53, 42, 41, 40, 39, 38, 37, 76, 75, 78, 79, 36, 33, 32, 29, 27, 25, 23, 20, 18, 16, 9, 7, 5]

nn_idx:  [67, 62, 58, 53, 50, 40, 37, 35, 38, 39, 14, 13, 8, 4, 1, 97, 94, 91, 90, 23, 22, 33, 31, 29, 30, 66, 68]
nn_idx:  [66, 61, 57, 53, 42, 40, 37, 35, 38, 39, 14, 12, 8, 5, 1, 98, 95, 92, 91, 24, 21, 19, 31, 30, 32, 67, 68]
nn_idx:  [62, 60, 57, 53, 42, 40, 37, 35, 38, 39, 14, 12, 8, 5, 2, 99, 95, 92, 91, 24, 22, 19, 31, 30, 32, 67, 68]
nn_idx:  [62, 60, 57, 53, 42, 40, 38, 36, 37, 39, 14, 11, 7, 5, 2, 99, 96, 92, 91, 24, 22, 19, 32, 30, 31, 67, 72]
nn_idx:  [62, 59, 56, 53, 42, 40, 38, 36, 37, 39, 14, 11, 7, 4, 2, 99, 96, 93, 25, 26, 22, 19, 32, 30, 31, 67, 72]
nn_idx:  [61, 59, 56, 53, 42, 40, 38, 36, 37, 39, 14, 11, 7, 4, 2, 99, 96, 93, 25, 24, 22, 19, 32, 30, 31, 67, 73]
nn_idx:  [61, 59, 56, 53, 42, 40, 38, 36, 37, 39, 14, 11, 7, 4, 2, 0, 97, 93, 25, 24, 22, 19, 32, 30, 31, 67, 73]
nn_idx:  [61, 59, 56, 53, 42, 40, 38, 36, 37, 39, 14, 11, 7, 4, 2, 0, 97, 94, 26, 25, 22, 19, 32, 30, 31, 67, 73]
nn_idx:  [61, 59, 56, 53, 42, 40, 38, 36, 37, 39, 14, 11, 7, 4, 2, 0, 97, 94, 26, 25, 22, 19, 32, 30, 31, 67, 73]

[0.6120192307692307, 0.7450900163666121, 0.9236706689536878, 0.7408637873754154, 0.7175757575757575]

7 nn_idx:  [1, 98, 95, 91, 87, 83, 80, 77, 73, 70, 66, 63, 59, 55, 52, 49, 45, 42, 37, 34, 31, 27, 24, 20, 17, 11, 8, 5, 2]
nn_idx:  [1, 98, 95, 91, 88, 84, 80, 77, 73, 70, 66, 63, 59, 55, 52, 49, 45, 42, 38, 34, 31, 27, 24, 20, 17, 11, 8, 4, 2]
nn_idx:  [1, 98, 95, 91, 88, 84, 80, 77, 73, 70, 66, 63, 59, 55, 52, 49, 45, 42, 38, 34, 31, 27, 24, 20, 17, 11, 8, 4, 2]

[0.6572115384615385, 0.9264214046822743, 0.7938504542278128, 0.7693920335429769, 0.737246680642907]

8 nn_idx:  [1, 99, 95, 91, 88, 85, 81, 77, 73, 71, 67, 64, 60, 56, 53, 47, 44, 41, 37, 34, 29, 26, 23, 19, 16, 11, 7, 5, 2]
nn_idx:  [1, 98, 95, 92, 88, 85, 81, 77, 74, 71, 67, 64, 60, 56, 53, 47, 45, 41, 37, 33, 30, 26, 23, 19, 16, 11, 8, 5, 2]
nn_idx:  [1, 98, 95, 92, 88, 85, 81, 77, 74, 71, 67, 64, 60, 56, 53, 48, 45, 41, 37, 33, 30, 26, 23, 19, 16, 11, 9, 5, 2]
nn_idx:  [1, 98, 95, 92, 88, 85, 81, 78, 74, 71, 67, 64, 60, 56, 53, 48, 45, 41, 37, 33, 30, 26, 23, 19, 16, 11, 9, 5, 2]
nn_idx:  [1, 98, 95, 92, 88, 85, 81, 78, 74, 71, 68, 64, 60, 56, 53, 48, 45, 41, 37, 33, 30, 26, 23, 19, 16, 11, 9, 5, 2]
nn_idx:  [1, 98, 95, 92, 88, 85, 81, 78, 74, 71, 68, 64, 60, 56, 53, 48, 45, 41, 37, 33, 30, 26, 23, 19, 16, 11, 9, 5, 2]

[0.5634615384615385, 0.9439799331103679, 0.8263041065482797, 0.8079911209766926, 0.7896781354051055]

9 nn_idx:  [97, 95, 93, 91, 89, 86, 85, 68, 66, 64, 60, 58, 53, 56, 57, 59, 55, 54, 52, 51, 50, 47, 46, 43, 42, 40, 38, 32, 30, 19, 20, 18, 65, 82, 80, 76, 75, 77, 8, 6, 4, 99, 98]
nn_idx:  [97, 95, 94, 91, 88, 86, 85, 68, 66, 64, 60, 58, 53, 54, 56, 57, 55, 52, 51, 50, 49, 47, 46, 44, 42, 41, 38, 32, 30, 19, 20, 18, 65, 71, 80, 76, 75, 74, 8, 6, 4, 99, 98]
nn_idx:  [97, 95, 94, 92, 88, 86, 85, 68, 66, 64, 60, 57, 53, 54, 56, 58, 55, 52, 51, 50, 49, 47, 46, 44, 42, 41, 38, 32, 30, 19, 20, 18, 17, 93, 80, 76, 74, 75, 8, 6, 4, 99, 98]
nn_idx:  [97, 95, 94, 92, 89, 86, 85, 68, 66, 64, 60, 57, 53, 54, 56, 58, 55, 52, 51, 50, 49, 47, 46, 44, 42, 41, 38, 32, 30, 19, 18, 20, 17, 88, 80, 76, 75, 74, 8, 6, 4, 99, 98]
nn_idx:  [97, 95, 94, 92, 88, 86, 85, 68, 66, 64, 60, 58, 53, 54, 56, 57, 55, 52, 51, 50, 49, 47, 46, 44, 42, 41, 38, 32, 30, 19, 18, 20, 17, 93, 80, 76, 75, 74, 8, 6, 4, 99, 98]
nn_idx:  [97, 95, 94, 92, 89, 86, 85, 68, 66, 64, 60, 57, 53, 54, 56, 58, 55, 52, 51, 50, 49, 47, 46, 44, 42, 41, 38, 32, 30, 19, 18, 20, 17, 88, 80, 76, 75, 74, 8, 6, 4, 99, 98]
nn_idx:  [97, 95, 94, 92, 88, 86, 85, 68, 66, 64, 60, 58, 53, 54, 56, 57, 55, 52, 51, 50, 49, 47, 46, 44, 42, 41, 38, 32, 30, 19, 18, 20, 17, 93, 80, 76, 75, 74, 8, 6, 4, 99, 98]
nn_idx:  [97, 95, 94, 92, 89, 86, 85, 68, 66, 64, 60, 57, 53, 54, 56, 58, 55, 52, 51, 50, 49, 47, 46, 44, 42, 41, 38, 32, 30, 19, 18, 20, 17, 88, 80, 76, 75, 74, 8, 6, 4, 99, 98]
nn_idx:  [97, 95, 94, 92, 88, 86, 85, 68, 66, 64, 60, 58, 53, 54, 56, 57, 55, 52, 51, 50, 49, 47, 46, 44, 42, 41, 38, 32, 30, 19, 18, 20, 17, 93, 80, 76, 75, 74, 8, 6, 4, 99, 98]
nn_idx:  [97, 95, 94, 92, 89, 86, 85, 68, 66, 64, 60, 57, 53, 54, 56, 58, 55, 52, 51, 50, 49, 47, 46, 44, 42, 41, 38, 32, 30, 19, 18, 20, 17, 88, 80, 76, 75, 74, 8, 6, 4, 99, 98]
nn_idx:  [97, 95, 94, 92, 88, 86, 85, 68, 66, 64, 60, 58, 53, 54, 56, 57, 55, 52, 51, 50, 49, 47, 46, 44, 42, 41, 38, 32, 30, 19, 18, 20, 17, 93, 80, 76, 75, 74, 8, 6, 4, 99, 98]
nn_idx:  [97, 95, 94, 92, 89, 86, 85, 68, 66, 64, 60, 57, 53, 54, 56, 58, 55, 52, 51, 50, 49, 47, 46, 44, 42, 41, 38, 32, 30, 19, 18, 20, 17, 88, 80, 76, 75, 74, 8, 6, 4, 99, 98]
nn_idx:  [97, 95, 94, 92, 88, 86, 85, 68, 66, 64, 60, 58, 53, 54, 56, 57, 55, 52, 51, 50, 49, 47, 46, 44, 42, 41, 38, 32, 30, 19, 18, 20, 17, 93, 80, 76, 75, 74, 8, 6, 4, 99, 98]
nn_idx:  [97, 95, 94, 92, 89, 86, 85, 68, 66, 64, 60, 57, 53, 54, 56, 58, 55, 52, 51, 50, 49, 47, 46, 44, 42, 41, 38, 32, 30, 19, 18, 20, 17, 88, 80, 76, 75, 74, 8, 6, 4, 99, 98]
nn_idx:  [97, 95, 94, 92, 88, 86, 85, 68, 66, 64, 60, 58, 53, 54, 56, 57, 55, 52, 51, 50, 49, 47, 46, 44, 42, 41, 38, 32, 30, 19, 18, 20, 17, 93, 80, 76, 75, 74, 8, 6, 4, 99, 98]
nn_idx:  [97, 95, 94, 92, 89, 86, 85, 68, 66, 64, 60, 57, 53, 54, 56, 58, 55, 52, 51, 50, 49, 47, 46, 44, 42, 41, 38, 32, 30, 19, 18, 20, 17, 88, 80, 76, 75, 74, 8, 6, 4, 99, 98]
nn_idx:  [97, 95, 94, 92, 88, 86, 85, 68, 66, 64, 60, 58, 53, 54, 56, 57, 55, 52, 51, 50, 49, 47, 46, 44, 42, 41, 38, 32, 30, 19, 18, 20, 17, 93, 80, 76, 75, 74, 8, 6, 4, 99, 98]
nn_idx:  [97, 95, 94, 92, 89, 86, 85, 68, 66, 64, 60, 57, 53, 54, 56, 58, 55, 52, 51, 50, 49, 47, 46, 44, 42, 41, 38, 32, 30, 19, 18, 20, 17, 88, 80, 76, 75, 74, 8, 6, 4, 99, 98]
nn_idx:  [97, 95, 94, 92, 88, 86, 85, 68, 66, 64, 60, 58, 53, 54, 56, 57, 55, 52, 51, 50, 49, 47, 46, 44, 42, 41, 38, 32, 30, 19, 18, 20, 17, 93, 80, 76, 75, 74, 8, 6, 4, 99, 98]
nn_idx:  [97, 95, 94, 92, 89, 86, 85, 68, 66, 64, 60, 57, 53, 54, 56, 58, 55, 52, 51, 50, 49, 47, 46, 44, 42, 41, 38, 32, 30, 19, 18, 20, 17, 88, 80, 76, 75, 74, 8, 6, 4, 99, 98]

nn_idx:  [95, 93, 91, 90, 87, 86, 85, 66, 65, 64, 61, 59, 57, 55, 52, 50, 47, 46, 44, 41, 36, 34, 11, 12, 32, 30, 28, 25, 23, 21, 19, 18, 17, 84, 71, 81, 79, 76, 77, 6, 4, 99, 97]
nn_idx:  [96, 94, 92, 90, 88, 86, 85, 68, 67, 64, 60, 59, 57, 55, 52, 50, 47, 45, 44, 41, 36, 34, 11, 12, 14, 30, 28, 26, 23, 21, 19, 18, 17, 83, 71, 81, 78, 76, 77, 6, 3, 99, 98]
nn_idx:  [96, 94, 92, 90, 88, 86, 85, 68, 67, 64, 60, 59, 57, 55, 52, 50, 47, 45, 44, 41, 36, 34, 11, 12, 14, 30, 28, 25, 23, 21, 19, 18, 17, 83, 71, 81, 78, 77, 76, 6, 3, 99, 98]
nn_idx:  [96, 94, 92, 90, 88, 86, 85, 68, 67, 64, 60, 59, 57, 55, 52, 50, 47, 45, 44, 41, 36, 34, 11, 12, 14, 30, 28, 25, 23, 21, 19, 18, 17, 83, 71, 81, 78, 77, 76, 6, 3, 99, 98]

nn_idx:  [1, 98, 95, 91, 87, 84, 80, 76, 71, 68, 64, 60, 56, 52, 47, 44, 41, 37, 33, 30, 25, 20, 16, 13, 9, 6, 2]
nn_idx:  [1, 98, 95, 91, 87, 83, 80, 76, 71, 68, 64, 60, 56, 52, 47, 44, 41, 37, 33, 30, 25, 20, 16, 13, 9, 6, 2]
nn_idx:  [1, 98, 95, 91, 87, 83, 80, 76, 71, 68, 64, 60, 56, 52, 47, 44, 41, 37, 33, 30, 25, 20, 16, 13, 9, 6, 2]

[0.5240384615384616, 0.7526881720430108, 0.6132075471698113, 0.6976744186046512, 0.8836363636363637]

10 nn_idx:  [3, 1, 99, 97, 94, 92, 90, 88, 86, 84, 81, 79, 76, 73, 71, 69, 67, 65, 61, 59, 57, 55, 53, 51, 49, 46, 43, 41, 39, 36, 33, 31, 29, 26, 24, 21, 19, 17, 15, 12, 9, 6, 4]
nn_idx:  [3, 1, 99, 97, 94, 92, 90, 88, 86, 84, 80, 79, 76, 73, 71, 69, 67, 64, 62, 59, 57, 55, 53, 51, 49, 45, 43, 41, 39, 36, 34, 31, 29, 26, 24, 21, 19, 17, 15, 12, 9, 6, 4]
nn_idx:  [4, 2, 99, 97, 94, 92, 90, 88, 86, 83, 80, 79, 76, 73, 71, 69, 66, 64, 62, 59, 57, 55, 53, 51, 49, 45, 43, 41, 39, 36, 34, 32, 29, 26, 24, 21, 19, 17, 15, 12, 9, 6, 5]
nn_idx:  [4, 2, 0, 97, 94, 92, 90, 88, 85, 83, 80, 79, 76, 73, 71, 69, 66, 64, 62, 59, 57, 55, 53, 51, 49, 45, 43, 41, 39, 37, 34, 32, 29, 26, 24, 21, 19, 17, 15, 12, 9, 7, 5]
nn_idx:  [4, 2, 0, 97, 94, 92, 90, 88, 85, 83, 80, 79, 76, 73, 71, 69, 66, 64, 62, 59, 57, 55, 53, 51, 49, 45, 43, 41, 39, 37, 34, 32, 29, 26, 24, 21, 19, 17, 15, 12, 9, 7, 5]

nn_idx:  [2, 99, 97, 95, 93, 91, 89, 87, 85, 83, 81, 69, 70, 66, 59, 57, 55, 53, 51, 43, 42, 41, 40, 39, 38, 37, 76, 75, 78, 79, 36, 33, 30, 29, 27, 26, 23, 21, 17, 15, 9, 6, 4]
nn_idx:  [1, 99, 97, 95, 93, 91, 89, 87, 84, 83, 81, 69, 68, 66, 60, 58, 56, 54, 52, 50, 42, 41, 40, 39, 38, 37, 76, 75, 78, 79, 36, 33, 32, 29, 27, 26, 23, 20, 19, 16, 9, 6, 3]
nn_idx:  [1, 99, 97, 95, 93, 91, 89, 87, 84, 83, 81, 69, 68, 66, 61, 59, 57, 55, 52, 50, 42, 41, 40, 39, 38, 37, 76, 75, 78, 79, 36, 33, 32, 30, 27, 26, 23, 20, 19, 16, 9, 6, 3]
nn_idx:  [1, 99, 97, 95, 93, 91, 89, 87, 84, 83, 81, 69, 68, 66, 61, 59, 57, 55, 53, 51, 42, 41, 40, 39, 38, 37, 76, 75, 78, 79, 36, 33, 32, 30, 27, 26, 23, 20, 19, 16, 9, 7, 3]
nn_idx:  [2, 0, 98, 95, 93, 91, 89, 87, 84, 83, 81, 69, 68, 66, 61, 59, 57, 55, 53, 51, 42, 41, 40, 39, 38, 37, 76, 75, 78, 79, 36, 33, 32, 30, 27, 26, 23, 20, 19, 16, 9, 7, 4]
nn_idx:  [2, 0, 98, 96, 93, 91, 89, 87, 84, 83, 81, 69, 68, 66, 61, 59, 57, 55, 53, 51, 42, 41, 40, 39, 38, 37, 76, 75, 78, 79, 36, 33, 32, 30, 27, 26, 23, 20, 19, 16, 9, 7, 4]
nn_idx:  [2, 0, 98, 96, 93, 91, 89, 87, 84, 83, 81, 69, 68, 66, 61, 59, 57, 55, 53, 51, 42, 41, 40, 39, 38, 37, 76, 75, 78, 79, 36, 33, 32, 30, 27, 26, 23, 20, 18, 16, 9, 7, 4]
nn_idx:  [2, 0, 98, 96, 93, 91, 89, 87, 84, 83, 81, 69, 68, 66, 61, 59, 57, 55, 53, 51, 42, 41, 40, 39, 38, 37, 76, 75, 78, 79, 36, 34, 32, 30, 27, 26, 23, 20, 18, 16, 9, 7, 4]
nn_idx:  [2, 0, 98, 96, 93, 91, 89, 87, 84, 83, 81, 69, 68, 66, 61, 59, 57, 55, 53, 51, 42, 41, 40, 39, 38, 37, 76, 75, 78, 79, 36, 34, 32, 30, 27, 26, 23, 20, 18, 16, 9, 7, 4]

[0.6173076923076923, 0.7100840336134453, 0.9275300171526587, 0.7346345514950166, 0.6834733893557423]

11 nn_idx:  [1, 98, 95, 94, 91, 86, 83, 84, 65, 64, 63, 58, 51, 57, 56, 59, 55, 54, 53, 52, 50, 48, 46, 44, 42, 40, 37, 32, 30, 17, 18, 19, 20, 16, 2, 74, 73, 8, 9, 7, 6, 3, 0]
nn_idx:  [99, 98, 95, 94, 91, 84, 83, 66, 65, 64, 63, 58, 51, 55, 57, 56, 54, 53, 52, 50, 49, 48, 46, 44, 42, 41, 37, 32, 30, 17, 19, 18, 20, 67, 75, 74, 73, 7, 9, 8, 6, 3, 1]
nn_idx:  [99, 97, 95, 93, 90, 84, 83, 66, 65, 63, 60, 58, 51, 26, 24, 23, 50, 54, 53, 52, 49, 48, 46, 44, 42, 40, 37, 32, 30, 27, 19, 18, 67, 71, 75, 74, 73, 7, 9, 6, 5, 3, 1]
nn_idx:  [98, 96, 94, 93, 91, 84, 83, 66, 65, 63, 61, 58, 51, 26, 24, 23, 50, 53, 52, 49, 48, 47, 45, 44, 42, 40, 37, 32, 30, 17, 19, 18, 20, 67, 75, 74, 73, 7, 9, 6, 5, 2, 99]
nn_idx:  [98, 95, 94, 92, 90, 84, 83, 66, 65, 63, 60, 58, 51, 26, 24, 23, 50, 53, 52, 49, 48, 47, 45, 43, 41, 40, 37, 32, 30, 17, 18, 19, 20, 67, 76, 74, 73, 7, 8, 6, 5, 2, 97]
nn_idx:  [97, 95, 94, 91, 90, 84, 83, 66, 65, 63, 60, 58, 51, 26, 24, 23, 50, 53, 52, 49, 48, 46, 45, 43, 41, 40, 37, 32, 30, 17, 19, 18, 20, 67, 78, 73, 74, 7, 8, 6, 5, 2, 98]
nn_idx:  [97, 95, 93, 91, 90, 84, 83, 66, 65, 63, 60, 58, 51, 26, 24, 23, 50, 53, 52, 49, 48, 46, 45, 43, 41, 40, 37, 32, 30, 17, 19, 18, 20, 67, 78, 73, 74, 7, 8, 6, 5, 2, 98]
nn_idx:  [97, 95, 93, 91, 90, 84, 83, 66, 65, 63, 60, 58, 51, 26, 24, 23, 50, 53, 52, 49, 48, 46, 45, 43, 41, 40, 37, 32, 30, 17, 19, 18, 20, 67, 78, 73, 74, 7, 8, 6, 5, 2, 98]

nn_idx:  [98, 95, 94, 91, 89, 84, 83, 66, 65, 63, 61, 59, 57, 55, 53, 51, 49, 46, 44, 42, 37, 36, 34, 33, 32, 29, 27, 25, 23, 21, 19, 17, 16, 18, 67, 77, 74, 73, 7, 6, 5, 3, 1]
nn_idx:  [98, 95, 93, 91, 89, 84, 83, 66, 65, 63, 61, 59, 57, 55, 53, 50, 49, 46, 44, 42, 37, 36, 34, 33, 32, 29, 27, 25, 23, 21, 19, 17, 18, 67, 70, 78, 74, 73, 7, 8, 6, 3, 1]
nn_idx:  [97, 95, 93, 91, 89, 85, 83, 66, 65, 63, 61, 59, 57, 55, 53, 50, 49, 46, 44, 42, 37, 36, 34, 33, 32, 29, 27, 25, 23, 21, 19, 17, 18, 67, 70, 78, 74, 73, 7, 8, 6, 3, 1]
nn_idx:  [97, 95, 93, 91, 87, 85, 83, 66, 65, 63, 61, 59, 57, 55, 53, 50, 49, 46, 44, 42, 37, 36, 35, 33, 32, 29, 27, 25, 23, 21, 19, 17, 18, 67, 70, 78, 74, 73, 7, 8, 6, 3, 1]
nn_idx:  [97, 95, 93, 91, 87, 85, 83, 66, 65, 63, 61, 59, 57, 55, 53, 50, 48, 46, 44, 42, 37, 36, 35, 33, 32, 29, 27, 25, 23, 21, 19, 17, 18, 67, 70, 78, 74, 73, 7, 8, 6, 3, 2]
nn_idx:  [97, 95, 93, 90, 87, 85, 83, 66, 65, 63, 61, 59, 57, 55, 52, 50, 48, 46, 44, 42, 37, 36, 35, 33, 32, 29, 27, 25, 23, 21, 19, 17, 18, 67, 70, 78, 74, 73, 7, 8, 6, 3, 2]
nn_idx:  [97, 95, 92, 90, 87, 85, 83, 66, 65, 63, 61, 59, 57, 55, 52, 50, 48, 46, 44, 42, 37, 36, 35, 33, 32, 29, 27, 25, 23, 21, 19, 17, 18, 67, 70, 78, 74, 73, 7, 8, 6, 3, 2]
nn_idx:  [97, 95, 92, 90, 87, 85, 83, 66, 65, 63, 61, 59, 57, 55, 52, 50, 48, 46, 44, 42, 37, 36, 35, 33, 32, 29, 27, 25, 23, 21, 19, 17, 18, 67, 70, 78, 74, 73, 7, 8, 6, 3, 2]

nn_idx:  [3, 1, 97, 93, 90, 79, 75, 74, 71, 67, 64, 60, 56, 52, 48, 44, 41, 38, 33, 29, 26, 19, 15, 11, 10, 7, 4]
nn_idx:  [3, 99, 96, 92, 90, 80, 75, 74, 71, 67, 64, 60, 55, 51, 48, 44, 41, 37, 32, 29, 25, 19, 17, 12, 10, 7, 4]
nn_idx:  [2, 99, 95, 91, 90, 80, 78, 74, 71, 67, 64, 60, 55, 51, 48, 45, 41, 37, 32, 29, 25, 20, 17, 13, 10, 7, 3]
nn_idx:  [1, 98, 95, 91, 90, 80, 78, 73, 70, 67, 64, 60, 55, 51, 48, 45, 41, 37, 32, 29, 25, 20, 17, 13, 10, 6, 3]
nn_idx:  [1, 98, 95, 91, 89, 81, 78, 73, 70, 67, 64, 60, 55, 51, 48, 45, 41, 37, 32, 29, 25, 20, 17, 13, 10, 6, 3]
nn_idx:  [1, 98, 95, 91, 89, 81, 78, 73, 70, 67, 64, 60, 55, 51, 48, 45, 41, 37, 32, 29, 25, 20, 17, 13, 10, 6, 3]

[0.5384615384615384, 0.7194444444444444, 0.6286449399656947, 0.787375415282392, 0.8606060606060606]

12 nn_idx:  [1, 95, 93, 91, 89, 87, 84, 82, 80, 77, 74, 73, 71, 67, 65, 64, 62, 60, 57, 54, 53, 50, 48, 46, 43, 41, 38, 36, 34, 32, 29, 27, 24, 22, 19, 16, 14, 13, 10, 7, 5, 4, 2]
nn_idx:  [1, 96, 93, 91, 89, 87, 84, 82, 80, 77, 74, 73, 71, 67, 65, 64, 62, 59, 57, 54, 53, 50, 48, 45, 43, 41, 38, 36, 34, 32, 29, 27, 24, 22, 19, 16, 14, 12, 10, 8, 5, 4, 2]
nn_idx:  [1, 96, 93, 91, 89, 87, 84, 82, 80, 77, 74, 73, 71, 67, 65, 64, 62, 59, 57, 54, 52, 50, 48, 45, 43, 41, 38, 36, 34, 32, 29, 27, 24, 22, 19, 16, 14, 12, 10, 8, 5, 4, 2]
nn_idx:  [1, 96, 93, 91, 89, 86, 84, 82, 80, 77, 74, 73, 71, 67, 65, 64, 62, 59, 57, 54, 52, 50, 48, 45, 43, 41, 38, 36, 34, 32, 29, 27, 24, 22, 19, 16, 14, 12, 10, 8, 5, 4, 2]
nn_idx:  [1, 96, 93, 91, 89, 86, 84, 82, 79, 77, 74, 73, 71, 67, 65, 64, 62, 59, 57, 54, 52, 50, 48, 45, 43, 41, 38, 36, 34, 32, 29, 27, 24, 22, 19, 16, 14, 12, 10, 8, 5, 4, 2]
nn_idx:  [1, 96, 93, 91, 89, 86, 84, 82, 79, 77, 74, 73, 71, 67, 65, 64, 62, 59, 57, 54, 52, 50, 48, 45, 43, 41, 38, 36, 34, 32, 29, 27, 24, 22, 19, 16, 14, 12, 10, 8, 5, 4, 2]

nn_idx:  [96, 93, 91, 89, 87, 85, 83, 80, 78, 76, 75, 64, 63, 62, 55, 53, 50, 48, 46, 38, 37, 36, 35, 34, 33, 32, 70, 71, 72, 73, 31, 29, 27, 24, 22, 20, 18, 16, 13, 10, 5, 4, 1]
nn_idx:  [96, 93, 91, 89, 87, 85, 83, 80, 78, 76, 75, 64, 63, 61, 55, 54, 51, 49, 47, 44, 37, 36, 35, 34, 33, 32, 70, 71, 72, 73, 31, 29, 27, 25, 23, 21, 18, 15, 13, 11, 5, 3, 1]
nn_idx:  [95, 93, 91, 89, 87, 85, 83, 80, 78, 76, 75, 64, 63, 61, 56, 54, 52, 50, 47, 45, 37, 36, 35, 34, 33, 32, 70, 69, 72, 73, 31, 29, 27, 25, 23, 21, 18, 15, 14, 12, 5, 3, 1]
nn_idx:  [95, 93, 91, 89, 87, 85, 83, 80, 78, 76, 75, 63, 62, 61, 56, 54, 52, 50, 48, 46, 37, 36, 35, 34, 33, 32, 70, 69, 72, 73, 31, 29, 27, 25, 23, 21, 19, 15, 14, 12, 5, 3, 1]
nn_idx:  [95, 93, 91, 89, 87, 85, 83, 80, 78, 76, 75, 63, 62, 61, 56, 54, 52, 50, 48, 46, 37, 36, 35, 34, 33, 32, 70, 69, 72, 73, 31, 29, 27, 25, 23, 21, 19, 15, 14, 12, 5, 3, 1]

nn_idx:  [61, 55, 52, 47, 37, 34, 32, 29, 33, 35, 9, 8, 4, 1, 93, 90, 87, 84, 83, 18, 17, 27, 26, 25, 28, 62, 63]
nn_idx:  [56, 54, 51, 47, 36, 34, 32, 31, 33, 35, 9, 7, 4, 1, 94, 91, 88, 85, 83, 18, 17, 14, 27, 25, 26, 61, 62]
nn_idx:  [56, 54, 50, 46, 37, 34, 32, 31, 33, 35, 9, 7, 4, 1, 95, 91, 89, 85, 83, 18, 17, 14, 27, 26, 28, 62, 64]
nn_idx:  [56, 53, 50, 46, 37, 34, 32, 31, 33, 35, 9, 7, 4, 1, 95, 92, 89, 85, 83, 18, 17, 14, 27, 26, 28, 62, 66]
nn_idx:  [55, 53, 50, 46, 37, 35, 33, 31, 32, 34, 9, 7, 4, 1, 95, 92, 89, 86, 20, 21, 17, 14, 27, 26, 28, 62, 66]
nn_idx:  [55, 53, 50, 46, 37, 34, 33, 31, 32, 35, 9, 7, 4, 1, 95, 92, 90, 86, 20, 21, 17, 14, 27, 26, 28, 62, 67]
nn_idx:  [55, 53, 50, 46, 37, 34, 33, 31, 32, 35, 9, 7, 4, 1, 95, 92, 90, 86, 20, 21, 17, 14, 27, 26, 28, 62, 67]

[0.5576923076923077, 0.7215932914046121, 0.9292452830188679, 0.7350498338870433, 0.7321212121212122]

13 nn_idx:  [5, 3, 2, 0, 96, 95, 93, 73, 72, 71, 70, 67, 60, 63, 64, 65, 62, 61, 59, 58, 57, 56, 54, 52, 50, 49, 48, 41, 39, 27, 29, 30, 76, 89, 88, 80, 82, 83, 17, 16, 13, 9, 6]
nn_idx:  [5, 3, 2, 0, 96, 94, 93, 73, 72, 71, 70, 67, 60, 63, 64, 65, 62, 61, 59, 58, 57, 56, 54, 52, 50, 49, 48, 41, 40, 27, 29, 30, 76, 89, 88, 80, 82, 83, 17, 16, 13, 9, 6]
nn_idx:  [5, 4, 2, 0, 96, 94, 93, 73, 72, 71, 70, 67, 60, 63, 64, 65, 62, 61, 59, 58, 57, 56, 54, 52, 50, 49, 48, 41, 40, 27, 29, 30, 76, 89, 88, 80, 82, 83, 17, 16, 13, 9, 6]
nn_idx:  [5, 4, 2, 0, 96, 94, 93, 73, 72, 71, 70, 67, 60, 63, 64, 65, 62, 61, 59, 58, 57, 56, 54, 52, 50, 49, 48, 41, 40, 27, 29, 30, 76, 89, 88, 80, 82, 83, 17, 16, 13, 9, 6]

nn_idx:  [4, 2, 0, 97, 95, 93, 74, 73, 72, 70, 68, 65, 64, 62, 59, 58, 56, 54, 52, 50, 43, 44, 21, 22, 23, 37, 36, 35, 33, 31, 30, 28, 29, 75, 77, 89, 87, 82, 83, 15, 13, 7, 5]
nn_idx:  [4, 2, 0, 96, 95, 93, 74, 73, 72, 70, 67, 65, 64, 62, 59, 58, 56, 55, 52, 50, 43, 44, 21, 22, 23, 37, 36, 35, 33, 31, 29, 28, 27, 30, 75, 89, 86, 82, 83, 15, 13, 8, 6]
nn_idx:  [5, 2, 1, 96, 94, 93, 74, 73, 72, 70, 67, 65, 64, 62, 59, 58, 56, 55, 52, 48, 43, 44, 21, 22, 23, 37, 36, 35, 33, 31, 29, 28, 27, 30, 75, 89, 86, 83, 84, 15, 13, 9, 6]
nn_idx:  [5, 3, 1, 96, 94, 93, 73, 72, 71, 70, 67, 65, 64, 62, 59, 58, 56, 55, 52, 42, 45, 44, 21, 22, 23, 39, 37, 35, 33, 31, 29, 28, 27, 74, 78, 89, 86, 83, 84, 15, 13, 9, 7]
nn_idx:  [5, 3, 1, 96, 94, 93, 73, 72, 71, 70, 67, 65, 64, 62, 59, 58, 56, 55, 52, 42, 44, 45, 21, 22, 23, 39, 37, 35, 33, 31, 29, 28, 27, 74, 78, 89, 86, 83, 84, 14, 13, 9, 7]
nn_idx:  [5, 3, 1, 96, 94, 93, 73, 72, 71, 70, 67, 65, 64, 62, 59, 58, 56, 55, 52, 42, 44, 45, 21, 22, 23, 39, 37, 35, 33, 31, 29, 28, 27, 74, 78, 89, 86, 83, 84, 14, 13, 9, 7]

nn_idx:  [10, 7, 3, 98, 95, 91, 88, 82, 78, 74, 71, 67, 62, 59, 56, 52, 49, 46, 42, 39, 34, 30, 26, 22, 19, 15, 11]
nn_idx:  [10, 7, 3, 98, 95, 91, 88, 82, 78, 74, 71, 67, 62, 59, 56, 52, 49, 46, 43, 39, 34, 30, 26, 22, 19, 15, 11]
nn_idx:  [10, 7, 3, 98, 95, 91, 88, 82, 78, 74, 71, 67, 62, 59, 56, 52, 49, 46, 43, 39, 34, 30, 26, 22, 19, 15, 11]

[0.5836538461538461, 0.7120098039215687, 0.6016295025728988, 0.6536544850498338, 0.8775757575757576]

14 nn_idx:  [1, 98, 95, 91, 88, 84, 80, 77, 73, 70, 67, 63, 59, 55, 52, 47, 43, 40, 37, 33, 30, 26, 22, 19, 15, 12, 8, 4, 2]
nn_idx:  [1, 98, 95, 91, 88, 84, 80, 77, 73, 70, 66, 63, 59, 55, 52, 47, 44, 40, 37, 33, 30, 26, 22, 19, 15, 12, 9, 4, 2]
nn_idx:  [1, 98, 95, 91, 88, 84, 80, 77, 73, 70, 66, 63, 59, 55, 52, 47, 44, 41, 37, 33, 30, 26, 22, 19, 15, 12, 9, 4, 2]
nn_idx:  [1, 98, 95, 91, 88, 84, 80, 77, 73, 70, 66, 63, 59, 55, 52, 47, 44, 41, 37, 33, 30, 26, 22, 19, 15, 12, 9, 4, 2]

[0.6995192307692308, 0.9423076923076923, 0.7399829497016197, 0.7152600170502984, 0.6871270247229326]

15 nn_idx:  [2, 0, 98, 96, 93, 91, 89, 87, 84, 82, 79, 77, 75, 72, 69, 67, 65, 63, 60, 58, 56, 53, 51, 48, 46, 44, 40, 38, 36, 34, 31, 29, 26, 24, 22, 19, 17, 15, 13, 10, 8, 5, 3]
nn_idx:  [2, 0, 98, 96, 93, 91, 89, 86, 84, 82, 79, 77, 75, 72, 69, 67, 65, 63, 60, 58, 56, 53, 51, 48, 46, 44, 40, 38, 36, 34, 31, 29, 26, 24, 22, 19, 17, 15, 13, 10, 8, 5, 3]
nn_idx:  [2, 0, 98, 96, 93, 91, 89, 86, 84, 82, 79, 77, 75, 72, 69, 67, 65, 63, 60, 58, 56, 53, 51, 48, 46, 44, 40, 38, 36, 34, 31, 29, 26, 24, 22, 19, 17, 15, 13, 10, 8, 5, 3]

nn_idx:  [0, 98, 96, 94, 92, 90, 87, 85, 82, 81, 80, 67, 68, 64, 58, 56, 54, 51, 49, 46, 40, 39, 38, 36, 35, 34, 37, 74, 76, 77, 75, 33, 18, 27, 24, 23, 20, 19, 15, 13, 7, 5, 3]
nn_idx:  [1, 98, 96, 94, 92, 90, 87, 85, 82, 81, 80, 67, 68, 64, 59, 56, 54, 53, 49, 47, 40, 39, 38, 37, 36, 35, 74, 75, 76, 77, 34, 31, 29, 27, 25, 23, 20, 18, 16, 13, 8, 5, 3]
nn_idx:  [1, 98, 96, 94, 92, 90, 87, 85, 82, 81, 80, 67, 66, 64, 59, 57, 55, 53, 50, 48, 40, 39, 38, 37, 36, 35, 74, 73, 76, 77, 34, 31, 29, 27, 25, 23, 20, 18, 16, 13, 8, 6, 3]
nn_idx:  [1, 98, 96, 94, 92, 90, 87, 85, 82, 81, 80, 67, 66, 64, 60, 58, 55, 53, 51, 48, 40, 39, 38, 37, 35, 36, 74, 73, 76, 77, 34, 31, 29, 28, 25, 23, 20, 18, 16, 13, 8, 6, 3]
nn_idx:  [1, 99, 96, 94, 92, 90, 87, 85, 82, 81, 80, 67, 66, 64, 60, 58, 55, 53, 51, 48, 40, 39, 38, 37, 35, 34, 73, 74, 76, 77, 75, 33, 29, 28, 25, 23, 20, 18, 16, 13, 8, 6, 3]
nn_idx:  [1, 99, 96, 94, 92, 90, 87, 85, 82, 81, 80, 67, 66, 64, 60, 58, 56, 53, 51, 49, 40, 39, 38, 37, 35, 34, 73, 74, 76, 77, 75, 33, 29, 28, 25, 23, 20, 18, 16, 13, 8, 6, 3]
nn_idx:  [1, 99, 96, 94, 92, 90, 87, 85, 82, 81, 80, 67, 66, 64, 60, 58, 56, 54, 51, 49, 40, 39, 38, 36, 35, 34, 73, 74, 76, 77, 75, 33, 29, 28, 25, 23, 20, 18, 16, 13, 8, 6, 3]
nn_idx:  [1, 99, 97, 94, 92, 90, 87, 85, 82, 81, 80, 67, 66, 64, 60, 58, 56, 54, 52, 49, 39, 38, 37, 36, 35, 34, 73, 74, 76, 77, 75, 33, 29, 28, 25, 23, 20, 18, 16, 14, 8, 6, 3]
nn_idx:  [1, 99, 97, 94, 92, 90, 87, 85, 82, 81, 80, 67, 66, 64, 60, 58, 56, 54, 52, 49, 39, 38, 37, 36, 35, 34, 73, 74, 76, 77, 75, 33, 29, 28, 25, 23, 20, 18, 16, 14, 8, 6, 3]

nn_idx:  [64, 59, 55, 49, 40, 37, 34, 32, 35, 36, 12, 11, 6, 3, 98, 95, 92, 89, 88, 21, 20, 17, 29, 27, 28, 63, 66]
nn_idx:  [61, 57, 54, 49, 39, 37, 35, 32, 34, 36, 12, 10, 6, 3, 99, 96, 93, 90, 89, 22, 20, 17, 29, 28, 30, 65, 68]
nn_idx:  [60, 57, 54, 49, 39, 37, 35, 32, 34, 36, 12, 9, 6, 3, 99, 96, 93, 90, 91, 23, 20, 17, 29, 28, 30, 65, 69]
nn_idx:  [59, 57, 53, 49, 39, 37, 35, 32, 34, 36, 12, 9, 5, 2, 99, 96, 93, 91, 23, 24, 20, 17, 29, 28, 30, 65, 70]
nn_idx:  [59, 56, 53, 49, 39, 37, 35, 33, 34, 36, 12, 9, 5, 2, 99, 96, 94, 91, 23, 24, 20, 17, 29, 28, 30, 65, 71]
nn_idx:  [58, 56, 53, 49, 39, 37, 35, 33, 34, 36, 12, 9, 5, 2, 99, 96, 94, 91, 23, 22, 20, 17, 29, 28, 30, 65, 71]
nn_idx:  [57, 56, 53, 49, 39, 37, 35, 33, 34, 36, 12, 9, 5, 2, 0, 97, 94, 92, 24, 23, 20, 17, 29, 28, 30, 65, 71]
nn_idx:  [57, 56, 53, 49, 39, 37, 35, 33, 34, 36, 12, 9, 5, 2, 0, 97, 94, 92, 24, 23, 20, 17, 29, 28, 30, 65, 71]

[0.5682692307692307, 0.7170868347338936, 0.9206689536878216, 0.7362956810631229, 0.7212121212121212]

16 nn_idx:  [1, 99, 97, 95, 92, 89, 87, 68, 67, 66, 64, 62, 55, 58, 59, 60, 57, 56, 54, 53, 52, 50, 48, 46, 44, 42, 40, 35, 33, 23, 22, 21, 70, 82, 81, 77, 78, 11, 12, 10, 8, 5, 2]
nn_idx:  [1, 99, 97, 95, 92, 89, 87, 68, 67, 66, 64, 62, 55, 58, 59, 60, 57, 56, 54, 53, 52, 50, 48, 46, 44, 42, 40, 35, 33, 23, 22, 21, 70, 82, 81, 78, 77, 11, 12, 10, 8, 5, 2]
nn_idx:  [1, 99, 97, 95, 92, 89, 87, 68, 67, 66, 64, 62, 55, 58, 59, 60, 57, 56, 54, 53, 52, 50, 48, 46, 44, 42, 40, 35, 33, 23, 22, 21, 70, 82, 81, 78, 77, 11, 12, 10, 8, 5, 2]

nn_idx:  [99, 97, 95, 93, 90, 87, 88, 68, 67, 65, 63, 60, 59, 56, 54, 52, 51, 48, 46, 44, 38, 37, 15, 16, 17, 32, 30, 28, 26, 24, 23, 21, 22, 86, 73, 82, 79, 78, 9, 10, 7, 5, 1]
nn_idx:  [99, 97, 95, 93, 90, 87, 88, 68, 67, 65, 63, 60, 59, 56, 54, 52, 50, 48, 46, 44, 38, 37, 15, 16, 17, 32, 30, 28, 26, 24, 22, 21, 20, 85, 74, 82, 79, 78, 9, 8, 7, 5, 1]
nn_idx:  [99, 98, 95, 93, 90, 88, 87, 68, 67, 65, 63, 60, 59, 56, 54, 52, 50, 48, 46, 44, 38, 37, 15, 16, 17, 32, 30, 28, 26, 24, 22, 21, 20, 85, 74, 82, 79, 78, 9, 8, 7, 5, 1]
nn_idx:  [99, 98, 95, 93, 90, 88, 87, 68, 67, 65, 63, 60, 59, 56, 54, 52, 50, 48, 46, 44, 38, 37, 15, 16, 17, 32, 30, 28, 26, 24, 22, 21, 20, 85, 74, 82, 79, 78, 9, 8, 7, 5, 1]

nn_idx:  [5, 2, 98, 94, 90, 85, 81, 77, 74, 68, 65, 62, 58, 54, 50, 46, 43, 39, 36, 32, 28, 23, 19, 15, 13, 10, 7]
nn_idx:  [5, 1, 98, 94, 90, 85, 82, 77, 74, 68, 65, 62, 57, 53, 50, 46, 43, 40, 36, 32, 28, 23, 20, 15, 13, 10, 7]
nn_idx:  [5, 1, 97, 93, 90, 85, 82, 77, 73, 68, 65, 62, 57, 53, 50, 46, 43, 40, 36, 32, 28, 23, 20, 15, 13, 10, 7]
nn_idx:  [5, 1, 97, 93, 90, 86, 82, 77, 73, 68, 65, 62, 57, 53, 50, 46, 43, 40, 36, 32, 28, 23, 20, 15, 13, 10, 7]
nn_idx:  [5, 1, 97, 93, 90, 86, 82, 77, 73, 68, 65, 62, 57, 53, 50, 46, 43, 40, 36, 32, 28, 23, 20, 15, 13, 10, 6]
nn_idx:  [5, 1, 97, 93, 90, 86, 82, 77, 73, 68, 65, 62, 57, 53, 50, 46, 43, 40, 36, 32, 28, 23, 20, 17, 13, 10, 6]
nn_idx:  [5, 1, 97, 93, 90, 86, 82, 76, 73, 68, 65, 62, 57, 53, 50, 46, 43, 40, 36, 32, 28, 23, 20, 17, 13, 10, 6]
nn_idx:  [5, 0, 97, 93, 90, 86, 82, 76, 73, 68, 65, 62, 57, 53, 50, 46, 43, 40, 36, 32, 28, 24, 20, 17, 13, 10, 6]
nn_idx:  [4, 0, 97, 93, 90, 86, 82, 76, 73, 68, 65, 62, 57, 53, 50, 46, 43, 40, 36, 32, 28, 24, 20, 17, 13, 10, 6]
nn_idx:  [4, 0, 97, 93, 90, 86, 82, 76, 73, 68, 65, 62, 57, 53, 50, 46, 43, 40, 36, 32, 28, 24, 20, 17, 13, 9, 6]
nn_idx:  [4, 0, 97, 93, 90, 86, 82, 76, 73, 68, 65, 61, 57, 53, 50, 46, 43, 40, 36, 32, 28, 24, 20, 17, 13, 9, 5]
nn_idx:  [4, 0, 97, 93, 90, 86, 82, 76, 73, 68, 65, 61, 57, 53, 50, 46, 43, 40, 36, 32, 28, 24, 20, 17, 13, 9, 5]

[0.5980769230769231, 0.7391304347826086, 0.66852487135506, 0.7495847176079734, 0.8763636363636363]

17 [0.6302884615384615, 0.9755244755244755, 0.7805944055944056, 0.7447552447552448, 0.7298951048951049]

18 nn_idx:  [4, 2, 0, 98, 95, 93, 91, 88, 87, 82, 81, 80, 70, 75, 76, 77, 74, 73, 71, 69, 68, 66, 63, 62, 60, 59, 58, 49, 46, 47, 31, 27, 25, 21, 19, 16, 15, 13, 11, 10, 9, 7, 5]
nn_idx:  [4, 2, 0, 98, 95, 93, 90, 88, 86, 82, 81, 80, 70, 75, 76, 77, 74, 73, 72, 71, 69, 66, 64, 62, 60, 59, 57, 49, 46, 32, 31, 27, 25, 22, 20, 17, 16, 14, 12, 11, 9, 6, 5]
nn_idx:  [4, 2, 0, 98, 95, 93, 90, 88, 86, 82, 81, 80, 74, 75, 76, 77, 73, 72, 71, 70, 69, 67, 64, 62, 60, 59, 57, 48, 46, 32, 33, 26, 25, 22, 20, 17, 16, 14, 12, 11, 9, 6, 5]
nn_idx:  [4, 2, 0, 97, 95, 93, 90, 88, 86, 82, 81, 80, 74, 75, 76, 77, 73, 72, 71, 70, 69, 67, 64, 62, 61, 59, 57, 48, 46, 32, 34, 26, 25, 22, 20, 17, 16, 14, 12, 11, 9, 6, 5]
nn_idx:  [3, 2, 0, 97, 95, 93, 90, 88, 86, 82, 81, 80, 74, 75, 76, 77, 73, 72, 71, 70, 69, 67, 65, 62, 61, 59, 57, 46, 47, 32, 34, 26, 25, 22, 20, 17, 16, 14, 12, 11, 9, 6, 4]
nn_idx:  [3, 2, 0, 97, 95, 93, 90, 88, 86, 82, 81, 80, 74, 75, 76, 77, 73, 72, 71, 70, 69, 67, 65, 62, 61, 59, 58, 46, 47, 32, 34, 26, 25, 22, 20, 17, 16, 14, 13, 11, 8, 6, 4]
nn_idx:  [3, 2, 0, 97, 95, 93, 90, 88, 86, 82, 81, 80, 74, 75, 76, 77, 73, 72, 71, 70, 69, 67, 65, 63, 61, 59, 58, 46, 47, 32, 34, 26, 25, 22, 20, 17, 16, 15, 13, 11, 8, 6, 4]
nn_idx:  [3, 1, 0, 97, 95, 93, 90, 88, 86, 82, 81, 80, 74, 75, 76, 77, 73, 72, 71, 70, 69, 67, 65, 63, 61, 59, 58, 46, 47, 32, 34, 26, 25, 22, 20, 17, 16, 15, 13, 11, 8, 6, 4]
nn_idx:  [3, 1, 99, 97, 95, 93, 90, 88, 86, 82, 81, 80, 74, 75, 76, 77, 73, 72, 71, 70, 69, 67, 65, 63, 61, 59, 58, 46, 47, 32, 34, 26, 25, 22, 20, 17, 16, 15, 13, 11, 8, 5, 4]
nn_idx:  [3, 1, 99, 97, 95, 93, 90, 88, 86, 82, 81, 80, 74, 75, 76, 77, 73, 72, 71, 70, 69, 67, 65, 63, 61, 59, 58, 46, 47, 32, 34, 26, 25, 22, 20, 17, 16, 15, 13, 12, 8, 5, 4]
nn_idx:  [3, 1, 99, 97, 95, 93, 90, 88, 86, 82, 81, 80, 74, 75, 76, 77, 73, 72, 71, 70, 69, 67, 65, 63, 61, 59, 58, 46, 47, 32, 34, 26, 25, 22, 20, 17, 16, 15, 13, 12, 8, 5, 4]

nn_idx:  [3, 0, 98, 96, 94, 92, 89, 87, 85, 82, 80, 78, 75, 73, 70, 68, 66, 64, 62, 59, 57, 54, 53, 50, 48, 46, 44, 41, 38, 36, 32, 30, 28, 26, 23, 20, 18, 16, 13, 11, 9, 7, 4]
nn_idx:  [3, 0, 98, 96, 94, 92, 89, 87, 85, 82, 80, 78, 75, 73, 70, 68, 66, 63, 62, 59, 57, 55, 53, 50, 48, 46, 44, 41, 38, 36, 33, 30, 28, 26, 23, 20, 18, 16, 13, 11, 9, 7, 4]
nn_idx:  [3, 0, 98, 96, 94, 92, 89, 87, 85, 82, 80, 78, 75, 73, 70, 68, 66, 63, 62, 59, 57, 55, 53, 50, 48, 46, 44, 41, 38, 36, 33, 30, 28, 26, 23, 20, 18, 16, 13, 11, 9, 7, 4]

[0.5600961538461539, 0.6913525498891353, 0.7212692967409948, 0.9194352159468439, 0.6745011086474502]

19 nn_idx:  [98, 96, 94, 92, 90, 88, 85, 83, 80, 78, 75, 73, 70, 66, 65, 63, 61, 58, 56, 54, 52, 50, 48, 46, 45, 43, 39, 36, 34, 31, 28, 25, 23, 20, 18, 15, 14, 11, 10, 7, 5, 2, 99]
nn_idx:  [98, 96, 94, 92, 90, 88, 85, 82, 80, 78, 75, 72, 69, 66, 64, 63, 61, 58, 56, 54, 52, 50, 48, 46, 45, 43, 39, 36, 34, 31, 28, 25, 23, 20, 18, 15, 14, 11, 10, 7, 5, 2, 99]
nn_idx:  [98, 96, 94, 92, 90, 88, 85, 82, 80, 78, 75, 72, 69, 66, 64, 63, 61, 58, 56, 54, 52, 50, 48, 46, 45, 43, 39, 36, 34, 31, 28, 25, 23, 20, 18, 15, 14, 11, 10, 7, 5, 2, 99]

nn_idx:  [96, 94, 93, 91, 89, 86, 84, 81, 78, 77, 76, 63, 62, 60, 55, 52, 51, 49, 46, 39, 38, 37, 36, 34, 33, 32, 69, 68, 71, 29, 28, 27, 14, 23, 21, 19, 17, 15, 12, 10, 4, 1, 98]
nn_idx:  [97, 94, 93, 91, 89, 86, 84, 81, 78, 77, 76, 63, 62, 60, 55, 53, 51, 50, 48, 46, 38, 37, 36, 34, 33, 32, 69, 68, 71, 72, 29, 28, 26, 24, 21, 19, 17, 15, 12, 10, 4, 1, 98]
nn_idx:  [97, 94, 93, 91, 89, 86, 84, 81, 78, 77, 76, 63, 62, 60, 55, 54, 52, 50, 48, 46, 38, 37, 36, 34, 33, 32, 69, 68, 71, 72, 29, 28, 26, 24, 21, 19, 17, 15, 12, 11, 5, 1, 98]
nn_idx:  [97, 94, 93, 91, 89, 86, 84, 81, 78, 77, 76, 63, 62, 60, 56, 54, 52, 50, 48, 46, 38, 37, 36, 34, 33, 32, 69, 68, 71, 72, 29, 28, 26, 24, 21, 19, 17, 14, 12, 11, 5, 1, 98]
nn_idx:  [97, 94, 93, 91, 89, 86, 84, 81, 78, 77, 76, 63, 62, 60, 56, 54, 52, 50, 49, 46, 38, 37, 36, 34, 33, 32, 68, 69, 71, 72, 29, 28, 26, 24, 21, 19, 17, 14, 12, 11, 5, 1, 98]
nn_idx:  [97, 94, 93, 91, 89, 86, 84, 81, 78, 77, 76, 63, 62, 60, 56, 54, 52, 51, 49, 46, 38, 37, 36, 34, 33, 32, 68, 69, 71, 72, 29, 28, 26, 24, 21, 19, 17, 14, 12, 11, 5, 1, 98]
nn_idx:  [97, 94, 93, 91, 89, 86, 84, 81, 78, 77, 76, 63, 62, 60, 56, 54, 52, 51, 49, 47, 38, 37, 36, 34, 33, 32, 68, 69, 71, 72, 29, 28, 26, 24, 21, 19, 17, 14, 12, 11, 5, 1, 98]
nn_idx:  [97, 94, 93, 91, 89, 86, 84, 81, 78, 77, 76, 63, 62, 60, 56, 54, 52, 51, 49, 47, 38, 37, 36, 34, 33, 32, 68, 69, 71, 72, 29, 28, 26, 24, 21, 19, 17, 14, 12, 11, 5, 1, 98]

nn_idx:  [60, 55, 51, 47, 38, 35, 32, 29, 31, 33, 8, 7, 3, 98, 94, 91, 89, 85, 84, 17, 16, 26, 25, 24, 27, 62, 61]
nn_idx:  [56, 54, 51, 47, 37, 34, 32, 29, 31, 33, 9, 6, 2, 98, 95, 92, 89, 86, 85, 18, 16, 14, 26, 24, 25, 61, 62]
nn_idx:  [56, 54, 51, 47, 37, 34, 32, 29, 31, 33, 9, 6, 2, 98, 95, 92, 90, 86, 85, 18, 16, 14, 25, 24, 26, 61, 64]
nn_idx:  [56, 53, 51, 47, 37, 34, 32, 29, 31, 33, 9, 6, 2, 98, 95, 93, 90, 87, 85, 18, 16, 14, 26, 24, 25, 61, 64]
nn_idx:  [55, 53, 51, 47, 37, 34, 32, 29, 31, 33, 9, 6, 2, 98, 95, 93, 91, 87, 86, 19, 16, 14, 26, 24, 25, 61, 64]
nn_idx:  [55, 53, 50, 47, 37, 34, 32, 29, 31, 33, 9, 6, 2, 98, 95, 93, 91, 88, 20, 19, 16, 14, 26, 24, 25, 61, 64]
nn_idx:  [55, 53, 50, 47, 37, 34, 32, 29, 31, 33, 9, 6, 2, 98, 96, 93, 91, 88, 20, 19, 16, 14, 26, 24, 25, 61, 65]
nn_idx:  [55, 53, 50, 47, 37, 34, 32, 29, 31, 33, 9, 6, 2, 98, 96, 93, 91, 88, 20, 19, 16, 14, 26, 24, 25, 61, 65]

[0.5557692307692308, 0.7969924812030076, 0.9343910806174958, 0.7333887043189369, 0.7284848484848485]

20 [0.5264423076923077, 0.9525166191832859, 0.8442545109211776, 0.8190883190883191, 0.8143399810066476]

21 nn_idx:  [1, 100, 98, 96, 94, 92, 76, 75, 74, 72, 69, 67, 65, 63, 61, 59, 57, 56, 53, 51, 46, 44, 20, 21, 40, 38, 36, 35, 33, 31, 29, 28, 27, 26, 97, 88, 86, 82, 83, 13, 11, 4, 2]
nn_idx:  [1, 100, 99, 97, 94, 92, 76, 75, 74, 72, 69, 67, 65, 63, 60, 59, 57, 56, 53, 51, 46, 43, 20, 21, 40, 38, 36, 35, 34, 31, 29, 27, 26, 28, 25, 98, 86, 83, 84, 12, 10, 6, 3]
nn_idx:  [2, 0, 99, 97, 94, 92, 76, 75, 74, 72, 69, 66, 64, 63, 60, 59, 57, 56, 53, 51, 45, 43, 20, 21, 40, 38, 37, 35, 34, 31, 29, 27, 26, 28, 25, 1, 85, 83, 84, 12, 10, 8, 3]
nn_idx:  [2, 0, 100, 98, 94, 92, 76, 75, 74, 72, 69, 66, 64, 62, 60, 59, 57, 56, 53, 51, 45, 43, 20, 21, 40, 38, 37, 35, 34, 31, 29, 27, 26, 28, 25, 1, 84, 83, 12, 13, 10, 8, 4]
nn_idx:  [3, 1, 100, 98, 94, 92, 76, 75, 74, 72, 69, 66, 64, 63, 60, 59, 57, 56, 53, 51, 45, 43, 20, 21, 40, 38, 37, 35, 34, 31, 29, 27, 26, 28, 25, 2, 84, 83, 13, 12, 10, 8, 4]
nn_idx:  [3, 1, 100, 98, 94, 92, 76, 75, 74, 72, 69, 66, 64, 63, 60, 59, 57, 56, 53, 51, 45, 43, 20, 21, 40, 38, 37, 35, 34, 31, 29, 26, 25, 28, 27, 2, 84, 83, 13, 12, 10, 8, 4]
nn_idx:  [3, 1, 100, 98, 94, 92, 76, 75, 74, 72, 69, 66, 64, 63, 60, 59, 57, 56, 53, 51, 45, 43, 20, 21, 40, 38, 37, 35, 34, 31, 29, 26, 25, 28, 27, 2, 84, 83, 13, 12, 10, 8, 4]

nn_idx:  [8, 3, 0, 97, 94, 91, 87, 82, 79, 75, 72, 69, 64, 61, 57, 53, 50, 46, 41, 38, 34, 29, 26, 21, 17, 13, 9]
nn_idx:  [8, 3, 0, 97, 94, 91, 87, 82, 79, 75, 73, 69, 64, 61, 57, 53, 50, 46, 41, 38, 34, 29, 26, 22, 17, 13, 9]
nn_idx:  [7, 3, 0, 97, 94, 91, 87, 82, 79, 75, 73, 69, 64, 61, 57, 53, 50, 46, 41, 38, 34, 29, 26, 22, 17, 13, 9]
nn_idx:  [7, 3, 0, 97, 94, 91, 87, 82, 79, 75, 73, 69, 64, 61, 57, 53, 50, 46, 41, 38, 34, 29, 26, 22, 17, 13, 9]

[0.6288461538461538, 0.736904761904762, 0.6916666666666667, 0.7134551495016611, 0.8896969696969697]


Score:  26.0

今まで通り、正しく点数が認識できました。
本当はここまで来るのにデバッグが大変でしたが…

ということで、デバッグ表示を外して、処理時間を見てみます。

処理時間計測

以下の2つはデバッグ表示を外したもの。

def calc_harupan(img, templates, svm):
    ctrs, resized_img = detect_candidate_contours(img, sat_th=50)
    # print('Number of candidates: ', len(ctrs))
    if len(ctrs) == 0:
        return 0.0, resized_img
    subctrs, _, _ = refine_contours(resized_img, ctrs)
    subctr_datasets = [contour_dataset(ctr) for ctr in subctrs]
    ########
    #### Simple code
    similarities = [get_similarities(d, templates)[0] for d in subctr_datasets]
    #### Code printing progress
#     similarities = []
#     for i,d in enumerate(subctr_datasets):
#         print(i, end=' ')
#         similarities += [get_similarities(d, templates)[0]]
#         print(similarities[-1])
#         print('')
#     print('')
    #######
    _, result = svm.predict(np.array(similarities, 'float32'))
    result = result.astype('int')
    score = 0.0
    texts = {0:'0', 1:'1', 2:'2', 3:'3', 5:'.5'}
    font = cv2.FONT_HERSHEY_SIMPLEX
    for res, ctr in zip(result, ctrs):
        if res[0] == 5:
            score += 0.5
        elif res[0] != -1:
            score += res[0]
        
        # Annotating recognized numbers for confirmation
        if res[0] != -1:
            resized_img = cv2.drawContours(resized_img, [ctr], -1, (0,255,0), 3)
            x,y,_,_ = cv2.boundingRect(ctr)
            resized_img = cv2.putText(resized_img, texts[res[0]], (x,y), font, 1, (230,230,0), 5)
    return score, resized_img

# Find optimum affine matrix using ICP algorithm
# src_pts: ndarray, shape is (n_s,2) (n_s: number of points)
# dst_pts: ndarray, shape is (n_d,2) (n_d: number of points, n_d should be larger or equal to n_s)
# initial_matrix: ndarray, shape is (2,3)
# search_range: float number, 0.0 ~ 1.0, the range to search nearest neighbor, 1.0 -> Search in all dst_pts
def icp_sub(src_pts, dst_pts, max_iter=20, initial_matrix=np.array([[1.0, 0.0, 0.0], [0.0, 1.0, 0.0]]), search_range=0.5):
    default_affine_matrix = np.array([[1.0, 0.0, 0.0], [0.0, 1.0, 0.0]])
    n_dst = dst_pts.shape[0]
    n_src = src_pts.shape[0]
    if n_dst < n_src:
        # print("icp: Insufficient destination points")
        return default_affine_matrix, False
    if initial_matrix.shape != (2,3):
        print("icp: Illegal shape of initial_matrix")
        return default_affine_matrix, False
    n_search = int(n_dst*search_range)
    M = initial_matrix
    # Store indices of the nearest neighbor point of dst_pts to the converted point of src_pts
    nn_idx = []
    converged = False
    for i in range(max_iter):
        nn_idx_tmp = []
        dst_pts_list = [p for p in dst_pts]
        idx_list = list(range(0,dst_pts.shape[0]))
        first_pt = True
        for p in src_pts:
            # Convert source point with current conversion matrix
            p2 = M @ np.array([p[0], p[1], 1])
            if first_pt:
                # First point should be searched in all destination points
                idx, _ = find_nearest_neighbor(dst_pts_list, p2)
                first_pt = False
            else:
                # Search nearest neighbor point in specified range around the last point
                n = int(min(n_search/2, len(idx_list)/2))
                s = max(len(idx_list) + last_idx - n, 0)
                e = min(len(idx_list) + last_idx + n, 3*len(idx_list))
                pts = (dst_pts_list + dst_pts_list + dst_pts_list)[s:e]
                idx, _ = find_nearest_neighbor(pts, p2)
                # The index acquired above is counted from 's', so actual index must be recovered
                idx = (idx + s) % len(idx_list)
            nn_idx_tmp += [idx_list[idx]]
            last_idx = idx
            del dst_pts_list[idx]
            del idx_list[idx]
        if nn_idx != [] and nn_idx == nn_idx_tmp:
            converged = True
            break
        dst_pts2 = np.zeros_like(src_pts)
        for j,idx in enumerate(nn_idx_tmp):
            dst_pts2[j,:] = dst_pts[idx,:]
        M = estimate_affine_2d(src_pts, dst_pts2)
        nn_idx = nn_idx_tmp
    return M, converged

以下は、timeitで処理時間計測を行う関数。

import timeit

def test_harupan_timeit(img, templates, svm):
    score, result_img = calc_harupan(img, templates, svm)
    n_loop = 5
    result = timeit.timeit('calc_harupan(img, templates, svm)', globals=globals(), number=n_loop)
    print('Score: ', score)
    print('Average process time: ', result/n_loop)
    plt.figure(figsize=(6.4,4.8), dpi=200)
    plt.imshow(cv2.cvtColor(result_img, cv2.COLOR_BGR2RGB)), plt.xticks([]), plt.yticks([])
    plt.show()
    return result/n_loop

今まで使った他の画像も用意して、こちらでも確認を行います。

img2 = cv2.imread('harupan_190428_2.jpg')
img3 = cv2.imread('harupan_200317_1.jpg')
img4 = cv2.imread('harupan_210227_2.jpg')
img5 = cv2.imread('harupan_210402_1.jpg')
img6 = cv2.imread('harupan_210402_2.jpg')
img7 = cv2.imread('harupan_210414_1.jpg')
img8 = cv2.imread('harupan_220330_1.jpg')
img9 = cv2.imread('harupan_220330_2.jpg')

いざ実施

imgs = [img1, img2, img3, img4, img5, img6, img7, img8, img9]
templates_sel = [0,0,1,2,2,2,2,2,2]

ts = []
for img, sel in zip(imgs, templates_sel):
    if sel == 0:
        templates = templates2019
    elif sel == 1:
        templates = templates2020
    else:
        templates = templates2021
    ts += [test_harupan_timeit(img, templates, svm)]

for t in ts:
    print(t)

Score:  26.0
Average process time:  2.49592544

Score:  26.0
Average process time:  2.86796598

Score:  25.0
Average process time:  1.5119802799999988

Score:  21.5
Average process time:  1.0334361200000004

Score:  28.0
Average process time:  2.0390738799999992

Score:  28.0
Average process time:  1.8528492200000017

Score:  25.0
Average process time:  1.7158804799999985

Score:  24.5
Average process time:  2.49107612

Score:  26.0
Average process time:  2.254828359999999

2.49592544
2.86796598
1.5119802799999988
1.0334361200000004
2.0390738799999992
1.8528492200000017
1.7158804799999985
2.49107612
2.254828359999999

点数の認識はおよそ正しくできていますが、ときどき誤認識があり、また、前回と比較すると、誤認識が増えているようでした。

処理時間は、前回の計測結果が

8.822 vs  7.391 vs 3.412
8.280 vs  6.903 vs 3.605
6.180 vs  3.850 vs 1.558
5.870 vs  2.600 vs 1.461
4.848 vs  3.601 vs 2.703
4.439 vs  3.107 vs 2.976
4.840 vs  3.779 vs 1.884
3.853 vs  3.112 vs 2.917
6.385 vs  5.484 vs 2.431

という感じ(一番右が最終結果)だったので、速くなるものはそれなりに速くなりましたが、それほど変わらないものもあり。

探索範囲を変えてみる

icp関数のsearch_range引数は、上記では0.5にしましたが、これを小さくして処理時間を確認してみたいと思います。
0.25、つまり全体の1/4ぐらいの範囲での探索で。

def icp(src_pts, dst_pts, max_iter=20, initial_matrix=np.array([[1.0, 0.0, 0.0], [0.0, 1.0, 0.0]])):
    search_range = 0.25
    return icp_sub(src_pts, dst_pts, max_iter=max_iter, initial_matrix=initial_matrix, search_range=search_range)

ts2 = []
for img, sel in zip(imgs, templates_sel):
    if sel == 0:
        templates = templates2019
    elif sel == 1:
        templates = templates2020
    else:
        templates = templates2021
    ts2 += [test_harupan_timeit(img, templates, svm)]

for t1,t2 in zip(ts, ts2):
    print('{:.3f} vs '.format(t1), '{:.3f}'.format(t2))

Score:  26.0
Average process time:  2.2573845800000014

Score:  26.0
Average process time:  2.5589878199999987

Score:  25.0
Average process time:  1.2015617599999984

Score:  21.5
Average process time:  0.8892099600000052

Score:  28.0
Average process time:  1.5628163000000028

Score:  28.0
Average process time:  1.5231709799999975

Score:  25.0
Average process time:  1.172573319999998

Score:  25.5
Average process time:  1.5129594800000006

Score:  26.0
Average process time:  1.5159665599999983

2.496 vs  2.257
2.868 vs  2.559
1.512 vs  1.202
1.033 vs  0.889
2.039 vs  1.563
1.853 vs  1.523
1.716 vs  1.173
2.491 vs  1.513
2.255 vs  1.516

点数認識のほうはそれほど変わらず。処理時間は、最後の2つは比較的速くなり、それ以外はあまり変わっていません。
もう1パターンくらいやってみる。
0.1ぐらいでどうか。

def icp(src_pts, dst_pts, max_iter=20, initial_matrix=np.array([[1.0, 0.0, 0.0], [0.0, 1.0, 0.0]])):
    search_range = 0.1
    return icp_sub(src_pts, dst_pts, max_iter=max_iter, initial_matrix=initial_matrix, search_range=search_range)

ts3 = []
for img, sel in zip(imgs, templates_sel):
    if sel == 0:
        templates = templates2019
    elif sel == 1:
        templates = templates2020
    else:
        templates = templates2021
    ts3 += [test_harupan_timeit(img, templates, svm)]

for t1,t2,t3 in zip(ts, ts2, ts3):
    print('{:.3f} vs '.format(t1), '{:.3f} vs '.format(t2), '{:3f}'.format(t3))

Score:  25.5
Average process time:  1.581251139999995

Score:  25.5
Average process time:  2.0109177600000065

Score:  25.0
Average process time:  0.9684285799999998

Score:  31.5
Average process time:  0.7069887800000061

Score:  36.5
Average process time:  1.2002911400000016

Score:  42.0
Average process time:  1.0803001199999926

Score:  26.0
Average process time:  1.0018412399999987

Score:  34.0
Average process time:  1.1885020799999892

Score:  26.5
Average process time:  1.2393877799999928

2.496 vs  2.257 vs  1.581251
2.868 vs  2.559 vs  2.010918
1.512 vs  1.202 vs  0.968429
1.033 vs  0.889 vs  0.706989
2.039 vs  1.563 vs  1.200291
1.853 vs  1.523 vs  1.080300
1.716 vs  1.173 vs  1.001841
2.491 vs  1.513 vs  1.188502
2.255 vs  1.516 vs  1.239388

処理は速くなっていますが、誤認識も結構増えてしまっている感じです。
0.1はやり過ぎかな。

以上

ICP処理の見直しで、ある程度処理を高速化することができました。
次回、PCカメラ画像からのリアルタイム処理に再チャレンジしてみたいと思います。
今度こそ最終回になるか…？

OpenCVやってみる - 43. 仕上げ(最終回)

今回は、春のパン祭りのシール点数集計の仕上げになるかと。

リアルタイム処理作成

PCのカメラ画像を連続的に取得して、点数集計を実施します。
今年は現状3枚の春のパン祭り台紙が用意できているので、これで確認します。

下準備

• スクリプト読み込み
• テンプレートデータ読み込み
• SVMデータ読み込み

を行います。
今回は、今年(2022年)の春のパン祭り点数の集計をやろうと思うので、2021年のテンプレートデータを使用します。

from harupan_data.harupan import *

svm = load_svm('harupan_data/harupan_svm_220412.dat')
templates2021= load_templates('harupan_data/templates2021.json')

PCカメラ画像は、前にやった通り、cv2.VideoCapture()でVideoCaptureオブジェクトを作成、read()で画像データ1枚取得、という手順になります。
今回はこれに加えて、

• 取得した画像に対して、点数集計処理を実施
• 結果を画像で表示(画像に合計点数も含める)
• 1枚処理するごとに、次の画像の取得、処理を行う。これを繰り返す。
• キーボード入力で終了

ということをやります。

Jupyter notebookで実行していますが、cv2.imshow()では、コマンドラインでやっていたときと同様、新しいウィンドウが開いて画像が表示されます。

def realtime_harupan():
    cap = cv2.VideoCapture(0)
    if not cap.isOpened():
        print('Cannot open camera')
        return
    else:
        print('Camera opened')
    while True:
        ret, frame = cap.read()
        if not ret:
            print('Can''t receive frame')
            cv2.destroyAllWindows()
            return
        else:
            score, result_img = calc_harupan(frame, templates2021, svm)
            score_text = str(score) + ' points'
            score_area = np.zeros((50, result_img.shape[1], 3), 'uint8')
            score_area = cv2.putText(score_area, score_text, (0,45), cv2.FONT_HERSHEY_SIMPLEX, 1, (255,255,0), 3)
            score_img = np.vstack((result_img, score_area))
            cv2.imshow('Result', score_img)
            k = cv2.waitKey(1)
            if k == ord('p'):
                cv2.waitKey(0)
            elif k == ord('e'):
                cap.release()
                return

realtime_harupan()

Camera opened

cv2.destroyAllWindows()

以下に実際に撮影しながら試してみた結果を示します。
スクリーン表示を取得して、GIFにしました。

まず1つ目。

結果としては、

• 更新が遅い…
  前回高速化はしましたが、それでもまだ実用的にはほど遠い…
• 期待通りに点数認識させるのが難しい
  照明の具合や台紙の角度、カメラからの距離などうまくいかないと、期待通りの点数認識ができませんでした。
  ノートPC本体に付属しているWebカメラなので、画面が見えるようにしつつ調整するのが余計難しく。
  USB接続のWebカメラならもうちょっとやりやすかったかな。

という感じで、結果は芳しくなく。

この春のパン祭り台紙で、もうちょっとうまくいかなかったときのGIFも載せます。

最初のほうは、PCから少し離してみていますが、そうすると点数がほとんど認識されていない。

2つ目の春のパン祭り台紙。

この台紙では、"2.5点"のシールが大量にあります。これがなかなか認識が難しいよう。

3つ目。

これも"2.5"点が難しい。
静止画は、奇跡的に正しく認識できた結果をとらえたもの。

以上

春のパン祭り集計は、ここまでにしておきます。
なかなか時間がかかったうえに、結果的には満足いく仕上がりにはならず。
きちんとしたものを作ろうと思ったら、今の知識でこれ以上いじくるよりは、また他を色々やって経験を積んでからがいいかと思います。やらないかもしれないけれど。
色々と勉強にはなったので、まあこんなところでしょう。

雑な文章と構成ですが、ご覧いただいた方いらっしゃいましたら、大変感謝です～。

一応最後に、最終版のスクリプトを掲載しておきます。
前回より微妙に修正があったりします。 (候補輪郭が全く検出されなかった場合の処理等)

######################################################
# Importing libraries
######################################################
from ctypes import resize
import cv2
import numpy as np
from matplotlib import pyplot as plt
import math
import copy
import random
import json

######################################################
# Detecting contours
######################################################
def reduce_resolution(img, res_th=800):
    h, w, chs = img.shape
    if h > res_th or w > res_th:
        k = float(res_th)/h if w > h else float(res_th)/w
    else:
        k = 1.0
    rtn_img = cv2.resize(img, None, fx=k, fy=k, interpolation=cv2.INTER_AREA)
    return rtn_img

def harupan_binarize(img, sat_th=100):
    hsv = cv2.cvtColor(img, cv2.COLOR_BGR2HSV)
    # Convert hue value (rotation, mask by saturation)
    hsv[:,:,0] = np.where(hsv[:,:,0] < 50, hsv[:,:,0]+180, hsv[:,:,0])
    hsv[:,:,0] = np.where(hsv[:,:,1] < sat_th, 0, hsv[:,:,0])
    # Thresholding with cv2.inRange()
    binary_img = cv2.inRange(hsv[:,:,0], 135, 190)
    return binary_img

def detect_candidate_contours(image, res_th=800, sat_th=100):
    img = reduce_resolution(image, res_th)
    binimg = harupan_binarize(img, sat_th)
    # Retrieve all points on the contours (cv2.CHAIN_APPROX_NONE)
    contours, hierarchy = cv2.findContours(binimg, cv2.RETR_TREE, cv2.CHAIN_APPROX_NONE)
    if len(contours) == 0:
        return contours, img
    # Pick up contours that have no parents
    indices = [i for i,hier in enumerate(hierarchy[0,:,:]) if hier[3] == -1]
    # Pick up contours that reside in above contours
    indices = [i for i,hier in enumerate(hierarchy[0,:,:]) if (hier[3] in indices) and (hier[2] == -1) ]
    contours = [contours[i] for i in indices]
    contours = [ctr for ctr in contours if cv2.contourArea(ctr) > float(res_th)*float(res_th)/4000]
    return contours, img

# image: Entire image containing multiple contours
# contours: Contours contained in "image" (Retrieved by cv2.findContours(), the origin is same as "image")
def refine_contours(image, contours):
    subctrs = []
    subimgs = []
    binimgs = []
    thresholds = []
    n_ctrs = []
    for ctr in contours:
        img, _ = create_contour_area_image(image, ctr)
        # Thresholding using G value in BGR format
        thresh, binimg = cv2.threshold(img[:,:,1], 0, 255, cv2.THRESH_BINARY + cv2.THRESH_OTSU)
        # Add black region around thresholded image, to detect contours correctly
        binimg = cv2.copyMakeBorder(binimg, 2,2,2,2, cv2.BORDER_CONSTANT, 0)
        ctrs2, _ = cv2.findContours(binimg, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_NONE)
        max_len = 0
        for ctr2 in ctrs2:
            if max_len <= ctr2.shape[0]:
                max_ctr = ctr2
                max_len = ctr2.shape[0]
        subctrs += [max_ctr]
        subimgs += [img]
        binimgs += [binimg]
        thresholds += [thresh]
        n_ctrs += [len(ctrs2)]
    debug_info = (binimgs, thresholds, n_ctrs)
    return subctrs, subimgs, debug_info

######################################################
# Auxiliary functions
######################################################
def create_contour_area_image(img, ctr):
    x,y,w,h = cv2.boundingRect(ctr)
    rtn_img = img[y:y+h,x:x+w,:].copy()
    rtn_ctr = ctr.copy()
    origin = np.array([x,y])
    for c in rtn_ctr:
        c[0,:] -= origin
    return rtn_img, rtn_ctr

# ctr: Should be output of create_contour_area_image() (Origin of points is the origin of bounding box)
# img_shape: Optional, tuple of (image_height, image_width), if omitted, calculated from ctr
def create_solid_contour(ctr, img_shape=(int(0),int(0))):
    if img_shape == (int(0),int(0)):
        _,_,w,h = cv2.boundingRect(ctr)
    else:
        h,w = img_shape
    img = np.zeros((h,w), 'uint8')
    img = cv2.drawContours(img, [ctr], -1, 255, -1)
    return img

# ctr: Should be output of create_contour_area_image() (Origin of points is the origin of bounding box)
def create_upright_solid_contour(ctr):
    ctr2 = ctr.copy()
    (cx,cy),(w,h),angle = cv2.minAreaRect(ctr2)
    M = cv2.getRotationMatrix2D((cx,cy), angle, 1)
    for i in range(ctr2.shape[0]):
        ctr2[i,0,:] = ( M @ np.array([ctr2[i,0,0], ctr2[i,0,1], 1]) ).astype('int')
    rect = cv2.boundingRect(ctr2)
    img = np.zeros((rect[3],rect[2]), 'uint8')
    ctr2 -= rect[0:2]
    M[:,2] -= rect[0:2]
    img = cv2.drawContours(img, [ctr2], -1, 255,-1)
    return img, M, ctr2


######################################################
# Dataset classes
######################################################
class contour_dataset:
    def __init__(self, ctr):
        self.ctr = ctr.copy()
        self.rrect = cv2.minAreaRect(ctr)
        self.box = cv2.boxPoints(self.rrect)
        self.solid = create_solid_contour(ctr)
        n = 100
        if n >= ctr.shape[0]:
            self.pts = np.array([p for p in ctr[:,0,:]])
        else:            
            r = n / ctr.shape[0]
            self.pts = np.zeros((100,2), 'int')
            pts = []
            for i in range(ctr.shape[0]):
                f = math.modf(i*r)[0] 
                if (f <= r/2) or (f > 1.0 - r/2):
                    pts += [ctr[i,0,:]]
            self.pts = np.array(pts)

class template_dataset:
    def __init__(self, ctr, num, selected_idx=[0]):
        self.ctr = ctr.copy()
        self.num = num
        self.rrect = cv2.minAreaRect(ctr)
        self.box = cv2.boxPoints(self.rrect)
        if num == 0:
            self.solid,_,_ = create_upright_solid_contour(ctr)
        else:
            self.solid = create_solid_contour(ctr)
        self.pts = np.array([ctr[idx,0,:] for idx in selected_idx])


######################################################
# ICP
######################################################
# pts: list of 2D points, or ndarray of shape (n,2)
# query: 2D point to find nearest neighbor
def find_nearest_neighbor(pts, query):
    min_distance_sq = float('inf')
    min_idx = 0
    for i, p in enumerate(pts):
        d = np.dot(query - p, query - p)
        if(d < min_distance_sq):
            min_distance_sq = d
            min_idx = i
    return min_idx, np.sqrt(min_distance_sq)

# src, dst: ndarray, shape is (n,2) (n: number of points)
def estimate_affine_2d(src, dst):
    n = min(src.shape[0], dst.shape[0])
    x = dst[0:n].flatten()
    A = np.zeros((2*n,6))
    for i in range(n):
        A[i*2,0] = src[i,0]
        A[i*2,1] = src[i,1]
        A[i*2,2] = 1
        A[i*2+1,3] = src[i,0]
        A[i*2+1,4] = src[i,1]
        A[i*2+1,5] = 1
    M = np.linalg.inv(A.T @ A) @ A.T @ x
    return M.reshape([2,3])

# Find optimum affine matrix using ICP algorithm
# src_pts: ndarray, shape is (n_s,2) (n_s: number of points)
# dst_pts: ndarray, shape is (n_d,2) (n_d: number of points, n_d should be larger or equal to n_s)
# initial_matrix: ndarray, shape is (2,3)
def icp(src_pts, dst_pts, max_iter=20, initial_matrix=np.array([[1.0, 0.0, 0.0], [0.0, 1.0, 0.0]])):
    default_affine_matrix = np.array([[1.0, 0.0, 0.0], [0.0, 1.0, 0.0]])
    if dst_pts.shape[0] < src_pts.shape[0]:
        # print("icp: Insufficient destination points")
        return default_affine_matrix, False
    if initial_matrix.shape != (2,3):
        print("icp: Illegal shape of initial_matrix")
        return default_affine_matrix, False
    M = initial_matrix
    # Store indices of the nearest neighbor point of dst_pts to the converted point of src_pts
    nn_idx = []
    for i in range(max_iter):
        nn_idx_tmp = []
        dst_pts_list = [p for p in dst_pts]
        idx_list = list(range(0,dst_pts.shape[0]))
        for p in src_pts:
            p2 = M @ np.array([p[0], p[1], 1])
            idx, d = find_nearest_neighbor(dst_pts_list, p2)
            nn_idx_tmp += [idx_list[idx]]
            del dst_pts_list[idx]
            del idx_list[idx]
        if nn_idx != [] and nn_idx == nn_idx_tmp:
            break
        dst_pts2 = np.zeros_like(src_pts)
        for j,idx in enumerate(nn_idx_tmp):
            dst_pts2[j,:] = dst_pts[idx,:]
        M = estimate_affine_2d(src_pts, dst_pts2)
        nn_idx = nn_idx_tmp
        if i == max_iter -1:
            return M, False
    return M, True


######################################################
# Calculating similarity and determining the number
######################################################
def binary_image_similarity(img1, img2):
    if img1.shape != img2.shape:
        print('binary_image_similarity: Different image size')
        return 0.0
    xor_img = cv2.bitwise_xor(img1, img2)
    return 1.0 - np.float(np.count_nonzero(xor_img)) / (img1.shape[0]*img2.shape[1])

# src, dst: contour_dataset or template_dataset (holding member variables box, solid)
def get_transform_by_rotated_rectangle(src, dst):
    # Rotated patterns are created when starting index is slided
    dst_box2 = np.vstack([dst.box, dst.box])
    max_similarity = 0.0
    max_converted_img = np.zeros((dst.solid.shape[1], dst.solid.shape[0]), 'uint8')
    for i in range(4):
        M = cv2.getAffineTransform(src.box[0:3], dst_box2[i:i+3])
        converted_img = cv2.warpAffine(src.solid, M, dsize=(dst.solid.shape[1], dst.solid.shape[0]), flags=cv2.INTER_NEAREST)
        similarity = binary_image_similarity(converted_img, dst.solid)
        if similarity > max_similarity:
            M_rtn = M
            max_similarity = similarity
            max_converted_img = converted_img
    return M_rtn, max_similarity, max_converted_img

def get_similarity_with_template(target_data, template_data, sim_th_high=0.95, sim_th_low=0.7):
    _,(w1,h1), _ = target_data.rrect
    _,(w2,h2), _ = template_data.rrect
    r = w1/h1 if w1 < h1 else h1/w1
    r = r * h2/w2 if w2 < h2 else r * w2/h2
    M, sim_init, _ = get_transform_by_rotated_rectangle(template_data, target_data)
    if sim_init > sim_th_high or sim_init < sim_th_low or r > 1.4 or r < 0.7:
        dsize = (template_data.solid.shape[1], template_data.solid.shape[0])
        flags = cv2.INTER_NEAREST|cv2.WARP_INVERSE_MAP
        converted_img = cv2.warpAffine(target_data.solid, M, dsize=dsize, flags=flags)
        return sim_init, converted_img
    M, _ = icp(template_data.pts, target_data.pts, initial_matrix=M)
    Minv = cv2.invertAffineTransform(M)
    converted_ctr = np.zeros_like(target_data.ctr)
    for i in range(target_data.ctr.shape[0]):
        converted_ctr[i,0,:] = (Minv[:,0:2] @ target_data.ctr[i,0,:]) + Minv[:,2]
    converted_img = create_solid_contour(converted_ctr, img_shape=template_data.solid.shape)
    val = binary_image_similarity(converted_img, template_data.solid)
    return val, converted_img

def get_similarity_with_template_zero(target_data, template_data):
    dsize = (template_data.solid.shape[1], template_data.solid.shape[0])
    converted_img = cv2.resize(target_data.solid, dsize=dsize, interpolation=cv2.INTER_NEAREST)
    val = binary_image_similarity(converted_img, template_data.solid)
    return val, converted_img

def get_similarities(target, templates):
    similarities = []
    converted_imgs = []
    for tmpl in templates:
        if tmpl.num == 0:
            sim,converted_img = get_similarity_with_template_zero(target, tmpl)
        else:
            sim,converted_img = get_similarity_with_template(target, tmpl)
        similarities += [sim]
        converted_imgs += [converted_img]
    return similarities, converted_imgs

def calc_harupan(img, templates, svm):
    ctrs, resized_img = detect_candidate_contours(img, sat_th=50)
    # print('Number of candidates: ', len(ctrs))
    if len(ctrs) == 0:
        return 0.0, resized_img
    subctrs, _, _ = refine_contours(resized_img, ctrs)
    subctr_datasets = [contour_dataset(ctr) for ctr in subctrs]
    ########
    #### Simple code
    similarities = [get_similarities(d, templates)[0] for d in subctr_datasets]
    #### Code printing progress
    # similarities = []
    # for i,d in enumerate(subctr_datasets):
    #     print(i, end=' ')
    #     similarities += [get_similarities(d, templates)[0]]
    # print('')
    ########
    _, result = svm.predict(np.array(similarities, 'float32'))
    result = result.astype('int')
    score = 0.0
    texts = {0:'0', 1:'1', 2:'2', 3:'3', 5:'.5'}
    font = cv2.FONT_HERSHEY_SIMPLEX
    for res, ctr in zip(result, ctrs):
        if res[0] == 5:
            score += 0.5
        elif res[0] != -1:
            score += res[0]
        
        # Annotating recognized numbers for confirmation
        if res[0] != -1:
            resized_img = cv2.drawContours(resized_img, [ctr], -1, (0,255,0), 3)
            x,y,_,_ = cv2.boundingRect(ctr)
            resized_img = cv2.putText(resized_img, texts[res[0]], (x,y), font, 1, (230,230,0), 5)
    return score, resized_img

######################################################
# Loading template data and SVM model
######################################################
def load_svm(filename):
    return cv2.ml.SVM_load(filename)

def load_templates(filename):
    with open(filename, mode='r') as f:
        load_data = json.load(f)
        templates_rtn = []
        for d in load_data:
            templates_rtn += [template_dataset(np.array(d['ctr']), d['num'], d['pts'])]
    return templates_rtn

OpenCVやってみる - 42. 処理の調整4

まだ春のパン祭りの続きですが、1点だけ処理の調整を試したいと思います。

-- 追記 --

今までmarkdown記法でブログを書いていると、ときどき画像がうまく表示されない(半角スペース4つのブロックの後に画像が表示されない)問題があって、前にはてなブログさんに修正の希望を出していましたが、どうも前回の記事を書いているときから、直っているようでした。
はてなさんありがとう～

変更内容

今まで見た点数輪郭では、"0"を除いて、その内部には輪郭を含みませんでした。 これを利用して、ICP処理を実施する前に候補輪郭を絞り、処理の高速化ができないか、試してみます。
"0"の輪郭は除外されてしまいますが、点数計算に影響ないので問題なし。

まずは今まで通りのスクリプト読み込みですが、どうも途中から関数を更新したりするのがうまくいかなかったので、全部コピペしておきます。

######################################################
# Importing libraries
######################################################
import cv2
import numpy as np
from matplotlib import pyplot as plt
import math
import copy
import random
import json

######################################################
# Detecting contours
######################################################
def reduce_resolution(img, res_th=800):
    h, w, chs = img.shape
    if h > res_th or w > res_th:
        k = float(res_th)/h if w > h else float(res_th)/w
    else:
        k = 1.0
    rtn_img = cv2.resize(img, None, fx=k, fy=k, interpolation=cv2.INTER_AREA)
    return rtn_img

def harupan_binarize(img, sat_th=100):
    hsv = cv2.cvtColor(img, cv2.COLOR_BGR2HSV)
    # Convert hue value (rotation, mask by saturation)
    hsv[:,:,0] = np.where(hsv[:,:,0] < 50, hsv[:,:,0]+180, hsv[:,:,0])
    hsv[:,:,0] = np.where(hsv[:,:,1] < sat_th, 0, hsv[:,:,0])
    # Thresholding with cv2.inRange()
    binary_img = cv2.inRange(hsv[:,:,0], 135, 190)
    return binary_img

def detect_candidate_contours(image, res_th=800, sat_th=100):
    img = reduce_resolution(image, res_th)
    binimg = harupan_binarize(img, sat_th)
    # Retrieve all points on the contours (cv2.CHAIN_APPROX_NONE)
    contours, hierarchy = cv2.findContours(binimg, cv2.RETR_TREE, cv2.CHAIN_APPROX_NONE)
    indices0 = [i for i,hier in enumerate(hierarchy[0,:,:]) if hier[3] == -1]
    indices1 = [i for i,hier in enumerate(hierarchy[0,:,:]) if hier[3] in indices0]
    contours1 = [contours[i] for i in indices1]
    contours1_filtered = [ctr for ctr in contours1 if cv2.contourArea(ctr) > float(res_th)*float(res_th)/4000]
    return contours1_filtered, img

# image: Entire image containing multiple contours
# contours: Contours contained in "image" (Retrieved by cv2.findContours(), the origin is same as "image")
def refine_contours(image, contours):
    subctrs = []
    subimgs = []
    binimgs = []
    thresholds = []
    n_ctrs = []
    for ctr in contours:
        img, _ = create_contour_area_image(image, ctr)
        # Thresholding using G value in BGR format
        thresh, binimg = cv2.threshold(img[:,:,1], 0, 255, cv2.THRESH_BINARY + cv2.THRESH_OTSU)
        # Add black region around thresholded image, to detect contours correctly
        binimg = cv2.copyMakeBorder(binimg, 2,2,2,2, cv2.BORDER_CONSTANT, 0)
        ctrs2, _ = cv2.findContours(binimg, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_NONE)
        max_len = 0
        for ctr2 in ctrs2:
            if max_len <= ctr2.shape[0]:
                max_ctr = ctr2
                max_len = ctr2.shape[0]
        subctrs += [max_ctr]
        subimgs += [img]
        binimgs += [binimg]
        thresholds += [thresh]
        n_ctrs += [len(ctrs2)]
    debug_info = (binimgs, thresholds, n_ctrs)
    return subctrs, subimgs, debug_info

######################################################
# Auxiliary functions
######################################################
def create_contour_area_image(img, ctr):
    x,y,w,h = cv2.boundingRect(ctr)
    rtn_img = img[y:y+h,x:x+w,:].copy()
    rtn_ctr = ctr.copy()
    origin = np.array([x,y])
    for c in rtn_ctr:
        c[0,:] -= origin
    return rtn_img, rtn_ctr

# ctr: Should be output of create_contour_area_image() (Origin of points is the origin of bounding box)
# img_shape: Optional, tuple of (image_height, image_width), if omitted, calculated from ctr
def create_solid_contour(ctr, img_shape=(int(0),int(0))):
    if img_shape == (int(0),int(0)):
        _,_,w,h = cv2.boundingRect(ctr)
    else:
        h,w = img_shape
    img = np.zeros((h,w), 'uint8')
    img = cv2.drawContours(img, [ctr], -1, 255, -1)
    return img

# ctr: Should be output of create_contour_area_image() (Origin of points is the origin of bounding box)
def create_upright_solid_contour(ctr):
    ctr2 = ctr.copy()
    (cx,cy),(w,h),angle = cv2.minAreaRect(ctr2)
    M = cv2.getRotationMatrix2D((cx,cy), angle, 1)
    for i in range(ctr2.shape[0]):
        ctr2[i,0,:] = ( M @ np.array([ctr2[i,0,0], ctr2[i,0,1], 1]) ).astype('int')
    rect = cv2.boundingRect(ctr2)
    img = np.zeros((rect[3],rect[2]), 'uint8')
    ctr2 -= rect[0:2]
    M[:,2] -= rect[0:2]
    img = cv2.drawContours(img, [ctr2], -1, 255,-1)
    return img, M, ctr2


######################################################
# Dataset classes
######################################################
class contour_dataset:
    def __init__(self, ctr):
        self.ctr = ctr.copy()
        self.rrect = cv2.minAreaRect(ctr)
        self.box = cv2.boxPoints(self.rrect)
        self.solid = create_solid_contour(ctr)
        self.pts = np.array([p for p in ctr[:,0,:]])

class template_dataset:
    def __init__(self, ctr, num, selected_idx=[0]):
        self.ctr = ctr.copy()
        self.num = num
        self.rrect = cv2.minAreaRect(ctr)
        self.box = cv2.boxPoints(self.rrect)
        if num == 0:
            self.solid,_,_ = create_upright_solid_contour(ctr)
        else:
            self.solid = create_solid_contour(ctr)
        self.pts = np.array([ctr[idx,0,:] for idx in selected_idx])


######################################################
# ICP
######################################################
# pts: list of 2D points, or ndarray of shape (n,2)
# query: 2D point to find nearest neighbor
def find_nearest_neighbor(pts, query):
    min_distance_sq = float('inf')
    min_idx = 0
    for i, p in enumerate(pts):
        d = np.dot(query - p, query - p)
        if(d < min_distance_sq):
            min_distance_sq = d
            min_idx = i
    return min_idx, np.sqrt(min_distance_sq)

# src, dst: ndarray, shape is (n,2) (n: number of points)
def estimate_affine_2d(src, dst):
    n = min(src.shape[0], dst.shape[0])
    x = dst[0:n].flatten()
    A = np.zeros((2*n,6))
    for i in range(n):
        A[i*2,0] = src[i,0]
        A[i*2,1] = src[i,1]
        A[i*2,2] = 1
        A[i*2+1,3] = src[i,0]
        A[i*2+1,4] = src[i,1]
        A[i*2+1,5] = 1
    M = np.linalg.inv(A.T @ A) @ A.T @ x
    return M.reshape([2,3])

# Find optimum affine matrix using ICP algorithm
# src_pts: ndarray, shape is (n_s,2) (n_s: number of points)
# dst_pts: ndarray, shape is (n_d,2) (n_d: number of points, n_d should be larger or equal to n_s)
# initial_matrix: ndarray, shape is (2,3)
def icp(src_pts, dst_pts, max_iter=20, initial_matrix=np.array([[1.0, 0.0, 0.0], [0.0, 1.0, 0.0]])):
    default_affine_matrix = np.array([[1.0, 0.0, 0.0], [0.0, 1.0, 0.0]])
    if dst_pts.shape[0] < src_pts.shape[0]:
        print("icp: Insufficient destination points")
        return default_affine_matrix, False
    if initial_matrix.shape != (2,3):
        print("icp: Illegal shape of initial_matrix")
        return default_affine_matrix, False
    M = initial_matrix
    # Store indices of the nearest neighbor point of dst_pts to the converted point of src_pts
    nn_idx = []
    for i in range(max_iter):
        nn_idx_tmp = []
        dst_pts_list = [p for p in dst_pts]
        idx_list = list(range(0,dst_pts.shape[0]))
        for p in src_pts:
            p2 = M @ np.array([p[0], p[1], 1])
            idx, d = find_nearest_neighbor(dst_pts_list, p2)
            nn_idx_tmp += [idx_list[idx]]
            del dst_pts_list[idx]
            del idx_list[idx]
        if nn_idx != [] and nn_idx == nn_idx_tmp:
            break
        dst_pts2 = np.zeros_like(src_pts)
        for j,idx in enumerate(nn_idx_tmp):
            dst_pts2[j,:] = dst_pts[idx,:]
        M = estimate_affine_2d(src_pts, dst_pts2)
        nn_idx = nn_idx_tmp
        if i == max_iter -1:
            return M, False
    return M, True


######################################################
# Calculating similarity and determining the number
######################################################
def binary_image_similarity(img1, img2):
    if img1.shape != img2.shape:
        print('binary_image_similarity: Different image size')
        return 0.0
    xor_img = cv2.bitwise_xor(img1, img2)
    return 1.0 - np.float(np.count_nonzero(xor_img)) / (img1.shape[0]*img2.shape[1])

# src, dst: contour_dataset or template_dataset (holding member variables box, solid)
def get_transform_by_rotated_rectangle(src, dst):
    # Rotated patterns are created when starting index is slided
    dst_box2 = np.vstack([dst.box, dst.box])
    max_similarity = 0.0
    max_converted_img = np.zeros((dst.solid.shape[1], dst.solid.shape[0]), 'uint8')
    for i in range(4):
        M = cv2.getAffineTransform(src.box[0:3], dst_box2[i:i+3])
        converted_img = cv2.warpAffine(src.solid, M, dsize=(dst.solid.shape[1], dst.solid.shape[0]), flags=cv2.INTER_NEAREST)
        similarity = binary_image_similarity(converted_img, dst.solid)
        if similarity > max_similarity:
            M_rtn = M
            max_similarity = similarity
            max_converted_img = converted_img
    return M_rtn, max_similarity, max_converted_img

def get_similarity_with_template(target_data, template_data, sim_th_high=0.95, sim_th_low=0.7):
    _,(w1,h1), _ = target_data.rrect
    _,(w2,h2), _ = template_data.rrect
    r = w1/h1 if w1 < h1 else h1/w1
    r = r * h2/w2 if w2 < h2 else r * w2/h2
    M, sim_init, _ = get_transform_by_rotated_rectangle(template_data, target_data)
    if sim_init > sim_th_high or sim_init < sim_th_low or r > 1.4 or r < 0.7:
        dsize = (template_data.solid.shape[1], template_data.solid.shape[0])
        flags = cv2.INTER_NEAREST|cv2.WARP_INVERSE_MAP
        converted_img = cv2.warpAffine(target_data.solid, M, dsize=dsize, flags=flags)
        return sim_init, converted_img
    M, _ = icp(template_data.pts, target_data.pts, initial_matrix=M)
    Minv = cv2.invertAffineTransform(M)
    converted_ctr = np.zeros_like(target_data.ctr)
    for i in range(target_data.ctr.shape[0]):
        converted_ctr[i,0,:] = (Minv[:,0:2] @ target_data.ctr[i,0,:]) + Minv[:,2]
    converted_img = create_solid_contour(converted_ctr, img_shape=template_data.solid.shape)
    val = binary_image_similarity(converted_img, template_data.solid)
    return val, converted_img

def get_similarity_with_template_zero(target_data, template_data):
    dsize = (template_data.solid.shape[1], template_data.solid.shape[0])
    converted_img = cv2.resize(target_data.solid, dsize=dsize, interpolation=cv2.INTER_NEAREST)
    val = binary_image_similarity(converted_img, template_data.solid)
    return val, converted_img

def get_similarities(target, templates):
    similarities = []
    converted_imgs = []
    for tmpl in templates:
        if tmpl.num == 0:
            sim,converted_img = get_similarity_with_template_zero(target, tmpl)
        else:
            sim,converted_img = get_similarity_with_template(target, tmpl)
        similarities += [sim]
        converted_imgs += [converted_img]
    return similarities, converted_imgs

def calc_harupan(img, templates, svm):
    ctrs, resized_img = detect_candidate_contours(img, sat_th=50)
    print('Number of candidates: ', len(ctrs))
    subctrs, _, _ = refine_contours(resized_img, ctrs)
    subctr_datasets = [contour_dataset(ctr) for ctr in subctrs]
    ########
    #### Simple code
    # similarities = [get_similarities(d, templates)[0] for d in subctr_datasets]
    #### Code printing progress
    similarities = []
    for i,d in enumerate(subctr_datasets):
        print(i, end=' ')
        similarities += [get_similarities(d, templates)[0]]
    print('')
    ########
    _, result = svm.predict(np.array(similarities, 'float32'))
    result = result.astype('int')
    score = 0.0
    texts = {0:'0', 1:'1', 2:'2', 3:'3', 5:'.5'}
    font = cv2.FONT_HERSHEY_SIMPLEX
    for res, ctr in zip(result, ctrs):
        if res[0] == 5:
            score += 0.5
        elif res[0] != -1:
            score += res[0]
        
        # Annotating recognized numbers for confirmation
        if res[0] != -1:
            resized_img = cv2.drawContours(resized_img, [ctr], -1, (0,255,0), 3)
            x,y,_,_ = cv2.boundingRect(ctr)
            resized_img = cv2.putText(resized_img, texts[res[0]], (x,y), font, 1, (230,230,0), 5)
    return score, resized_img

######################################################
# Loading template data and SVM model
######################################################
def load_svm(filename):
    return cv2.ml.SVM_load(filename)

def load_templates(filename):
    with open(filename, mode='r') as f:
        load_data = json.load(f)
        templates_rtn = []
        for d in load_data:
            templates_rtn += [template_dataset(np.array(d['ctr']), d['num'], d['pts'])]
    return templates_rtn

画像読み込みと、%matplotlib inlineのマジックコマンドも書いておきます。
学習済みSVM、テンプレートデータの読み込みも行います。

%matplotlib inline

img1 = cv2.imread('harupan_190428_1.jpg')
img2 = cv2.imread('harupan_190428_2.jpg')
img3 = cv2.imread('harupan_200317_1.jpg')
img4 = cv2.imread('harupan_210227_2.jpg')
img5 = cv2.imread('harupan_210402_1.jpg')
img6 = cv2.imread('harupan_210402_2.jpg')
img7 = cv2.imread('harupan_210414_1.jpg')
img8 = cv2.imread('harupan_220330_1.jpg')
img9 = cv2.imread('harupan_220330_2.jpg')

svm = cv2.ml.SVM_load('harupan_data/harupan_svm_220412.dat')
templates2019 = load_templates('harupan_data/templates2019_220412.json')
templates2020 = load_templates('harupan_data/templates2020_220412.json')
templates2021 = load_templates('harupan_data/templates2021_220412.json')

また、変更の効果の確認のため、一旦今まで通りの処理を各画像で実施します。
処理時間は、timeit.timeit()で測定します。

import timeit

def test_harupan_timeit(img, templates, svm):
    score, result_img = calc_harupan(img, templates, svm)
    n_loop = 5
    result = timeit.timeit('calc_harupan(img, templates, svm)', globals=globals(), number=n_loop)
    print('Score: ', score)
    print('Average process time: ', result/n_loop)
    plt.figure(figsize=(6.4,4.8), dpi=200)
    plt.imshow(cv2.cvtColor(result_img, cv2.COLOR_BGR2RGB)), plt.xticks([]), plt.yticks([])
    plt.show()
    return result/n_loop

imgs = [img1, img2, img3, img4, img5, img6, img7, img8, img9]
templates_sel = [0,0,1,2,2,2,2,2,2]

ts = []
for img, sel in zip(imgs, templates_sel):
    if sel == 0:
        templates = templates2019
    elif sel == 1:
        templates = templates2020
    else:
        templates = templates2021
    ts += [test_harupan_timeit(img, templates, svm)]

Number of candidates:  36
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 
Number of candidates:  36
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 
Number of candidates:  36
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 
Number of candidates:  36
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 
Number of candidates:  36
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 
Number of candidates:  36
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 
Score:  26.0
Average process time:  8.82181928

Number of candidates:  35
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 
Number of candidates:  35
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 
Number of candidates:  35
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 
Number of candidates:  35
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 
Number of candidates:  35
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 
Number of candidates:  35
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 
Score:  26.5
Average process time:  8.280414280000002

Number of candidates:  40
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 
Number of candidates:  40
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 
Number of candidates:  40
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 
Number of candidates:  40
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 
Number of candidates:  40
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 
Number of candidates:  40
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 
Score:  25.0
Average process time:  6.180490499999999

Number of candidates:  35
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 
Number of candidates:  35
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 
Number of candidates:  35
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 
Number of candidates:  35
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 
Number of candidates:  35
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 
Number of candidates:  35
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 
Score:  21.5
Average process time:  5.870072019999998

Number of candidates:  39
0 1 2 3 4 5 6 7 8 9 icp: Insufficient destination points
10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 
Number of candidates:  39
0 1 2 3 4 5 6 7 8 9 icp: Insufficient destination points
10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 
Number of candidates:  39
0 1 2 3 4 5 6 7 8 9 icp: Insufficient destination points
10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 
Number of candidates:  39
0 1 2 3 4 5 6 7 8 9 icp: Insufficient destination points
10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 
Number of candidates:  39
0 1 2 3 4 5 6 7 8 9 icp: Insufficient destination points
10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 
Number of candidates:  39
0 1 2 3 4 5 6 7 8 9 icp: Insufficient destination points
10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 
Score:  28.0
Average process time:  4.848178239999998

Number of candidates:  34
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 
Number of candidates:  34
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 
Number of candidates:  34
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 
Number of candidates:  34
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 
Number of candidates:  34
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 
Number of candidates:  34
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 
Score:  28.0
Average process time:  4.4394959000000025

Number of candidates:  22
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 
Number of candidates:  22
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 
Number of candidates:  22
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 
Number of candidates:  22
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 
Number of candidates:  22
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 
Number of candidates:  22
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 
Score:  25.0
Average process time:  4.83998542

Number of candidates:  25
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 
Number of candidates:  25
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 
Number of candidates:  25
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 
Number of candidates:  25
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 
Number of candidates:  25
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 
Number of candidates:  25
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 
Score:  26.0
Average process time:  3.852909180000006

Number of candidates:  24
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 
Number of candidates:  24
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 
Number of candidates:  24
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 
Number of candidates:  24
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 
Number of candidates:  24
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 
Number of candidates:  24
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 
Score:  26.0
Average process time:  6.385288779999996

for t in ts:
    print(t)

8.82181928
8.280414280000002
6.180490499999999
5.870072019999998
4.848178239999998
4.4394959000000025
4.83998542
3.852909180000006
6.385288779999996

スクリプト変更

スクリプトのうち、detect_candidate_contours()関数を以下のように変更します。

• 階層情報込みで輪郭検出 (今まで通り)
• 2段目の階層にある輪郭を取り出し (今まで通り)
• それより下の階層に輪郭を含まない輪郭を選択する

変更後、スクリプトの関数を上書きします。以下参照。

def detect_candidate_contours(image, res_th=800, sat_th=100):
    img = reduce_resolution(image, res_th)
    binimg = harupan_binarize(img, sat_th)
    # Retrieve all points on the contours (cv2.CHAIN_APPROX_NONE)
    contours, hierarchy = cv2.findContours(binimg, cv2.RETR_TREE, cv2.CHAIN_APPROX_NONE)
    # Pick up contours that have no parents
    indices = [i for i,hier in enumerate(hierarchy[0,:,:]) if hier[3] == -1]
    # Pick up contours that reside in above contours
    indices = [i for i,hier in enumerate(hierarchy[0,:,:]) if (hier[3] in indices) and (hier[2] == -1) ]
    contours = [contours[i] for i in indices]
    contours = [ctr for ctr in contours if cv2.contourArea(ctr) > float(res_th)*float(res_th)/4000]
    return contours, img

ts2 = []
for img, sel in zip(imgs, templates_sel):
    if sel == 0:
        templates = templates2019
    elif sel == 1:
        templates = templates2020
    else:
        templates = templates2021
    ts2 += [test_harupan_timeit(img, templates, svm)]

Number of candidates:  22
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 
Number of candidates:  22
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 
Number of candidates:  22
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 
Number of candidates:  22
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 
Number of candidates:  22
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 
Number of candidates:  22
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 
Score:  26.0
Average process time:  7.390565659999993

Number of candidates:  23
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 
Number of candidates:  23
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 
Number of candidates:  23
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 
Number of candidates:  23
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 
Number of candidates:  23
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 
Number of candidates:  23
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 
Score:  26.5
Average process time:  6.902831980000007

Number of candidates:  23
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 
Number of candidates:  23
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 
Number of candidates:  23
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 
Number of candidates:  23
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 
Number of candidates:  23
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 
Number of candidates:  23
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 
Score:  25.0
Average process time:  3.8497284000000036

Number of candidates:  25
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 
Number of candidates:  25
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 
Number of candidates:  25
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 
Number of candidates:  25
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 
Number of candidates:  25
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 
Number of candidates:  25
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 
Score:  19.5
Average process time:  2.6001762999999984

Number of candidates:  31
0 1 2 3 4 5 6 icp: Insufficient destination points
7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 
Number of candidates:  31
0 1 2 3 4 5 6 icp: Insufficient destination points
7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 
Number of candidates:  31
0 1 2 3 4 5 6 icp: Insufficient destination points
7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 
Number of candidates:  31
0 1 2 3 4 5 6 icp: Insufficient destination points
7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 
Number of candidates:  31
0 1 2 3 4 5 6 icp: Insufficient destination points
7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 
Number of candidates:  31
0 1 2 3 4 5 6 icp: Insufficient destination points
7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 
Score:  28.0
Average process time:  3.600558000000001

Number of candidates:  28
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 
Number of candidates:  28
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 
Number of candidates:  28
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 
Number of candidates:  28
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 
Number of candidates:  28
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 
Number of candidates:  28
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 
Score:  28.0
Average process time:  3.107207219999998

Number of candidates:  18
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 
Number of candidates:  18
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 
Number of candidates:  18
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 
Number of candidates:  18
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 
Number of candidates:  18
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 
Number of candidates:  18
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 
Score:  25.0
Average process time:  3.779431460000012

Number of candidates:  20
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 
Number of candidates:  20
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 
Number of candidates:  20
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 
Number of candidates:  20
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 
Number of candidates:  20
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 
Number of candidates:  20
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 
Score:  26.0
Average process time:  3.1119086799999875

Number of candidates:  20
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 
Number of candidates:  20
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 
Number of candidates:  20
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 
Number of candidates:  20
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 
Number of candidates:  20
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 
Number of candidates:  20
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 
Score:  26.0
Average process time:  5.48430937999999

for t1,t2 in zip(ts, ts2):
    print('{:.3f} vs '.format(t1), '{:.3f}'.format(t2))

8.822 vs  7.391
8.280 vs  6.903
6.180 vs  3.850
5.870 vs  2.600
4.848 vs  3.601
4.439 vs  3.107
4.840 vs  3.779
3.853 vs  3.112
6.385 vs  5.484

少し処理時間は短くなりましたが、劇的には改善せず。
あと4つ目の画像で1つ点数文字の認識が消えてしまった…

もうちょい処理を検討…

変更2

上の9画像を見ると、点数文字が比較的小さく写っているもののほうが処理時間が短いような。
ICPでのマッチング候補点数が少なくなることによるかと思われます。

ということで、ICPで使う輪郭点を間引きます。
contour_datasetsクラスのコンストラクタ内の処理を変更することになります。

どれくらい間引くかですが、前に選択したテンプレートで輪郭点数を見ると、1つの文字当たり100～200点くらいだったので、同程度あれば大丈夫だろうか。

間引き処理

100点に間引きをすることをターゲットにすると、

• 例えば353点あるとして、これが一列に等間隔(整数点位置)で並んでいるイメージ
• これを100点分の長さに圧縮する
• 圧縮後の各点について、一番近い整数点位置に対応付け
• 同じ整数点に複数の点が対応付けられるが、そのうち1つを代表で選ぶ

という感じ。
cv2.resize()で画像を縮小するときにcv2.INTER_NEARESTを選ぶのと同じようなことかと思われます。

class contour_dataset:
    def __init__(self, ctr):
        self.ctr = ctr.copy()
        self.rrect = cv2.minAreaRect(ctr)
        self.box = cv2.boxPoints(self.rrect)
        self.solid = create_solid_contour(ctr)
        n = 100
        if n >= ctr.shape[0]:
            self.pts = np.array([p for p in ctr[:,0,:]])
        else:            
            r = n / ctr.shape[0]
            self.pts = np.zeros((100,2), 'int')
            pts = []
            for i in range(ctr.shape[0]):
                f = math.modf(i*r)[0] 
                if (f <= r/2) or (f > 1.0 - r/2):
                    pts += [ctr[i,0,:]]
            self.pts = np.array(pts)
            print('points : ', ctr.shape[0], ' -> ', len(pts))

def test_harupan(img, templates, svm):
    score, result_img = calc_harupan(img, templates, svm)
    print('Score: ', score)
    plt.figure(figsize=(6.4,4.8), dpi=200)
    plt.imshow(cv2.cvtColor(result_img, cv2.COLOR_BGR2RGB)), plt.xticks([]), plt.yticks([])
    plt.show()

for img, sel in zip(imgs, templates_sel):
    if sel == 0:
        templates = templates2019
    elif sel == 1:
        templates = templates2020
    else:
        templates = templates2021
    test_harupan(img, templates, svm)

Number of candidates:  22
points :  110  ->  107
points :  128  ->  100
points :  123  ->  101
points :  125  ->  101
points :  201  ->  100
points :  132  ->  100
points :  199  ->  100
points :  135  ->  101
points :  123  ->  101
points :  123  ->  101
points :  201  ->  100
points :  126  ->  100
points :  210  ->  97
points :  133  ->  99
points :  137  ->  101
points :  199  ->  100
points :  126  ->  100
points :  142  ->  100
points :  212  ->  100
points :  223  ->  100
points :  134  ->  100
points :  139  ->  101
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 
Score:  26.0

Number of candidates:  23
points :  112  ->  100
points :  130  ->  97
points :  125  ->  101
points :  128  ->  100
points :  205  ->  99
points :  134  ->  100
points :  204  ->  100
points :  139  ->  101
points :  127  ->  100
points :  127  ->  100
points :  207  ->  100
points :  131  ->  100
points :  220  ->  100
points :  140  ->  100
points :  142  ->  100
points :  207  ->  100
points :  130  ->  97
points :  153  ->  101
points :  235  ->  101
points :  226  ->  99
points :  143  ->  100
points :  150  ->  100
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 
Score:  26.5

Number of candidates:  23
points :  146  ->  100
points :  144  ->  100
points :  169  ->  100
points :  143  ->  100
points :  143  ->  100
points :  160  ->  100
points :  169  ->  100
points :  208  ->  100
points :  145  ->  99
points :  145  ->  99
points :  144  ->  100
points :  145  ->  99
points :  148  ->  100
points :  144  ->  100
points :  145  ->  99
points :  203  ->  101
points :  144  ->  100
points :  199  ->  100
points :  143  ->  100
points :  200  ->  100
points :  162  ->  100
points :  149  ->  100
points :  150  ->  100
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 
Score:  25.0

Number of candidates:  25
points :  147  ->  99
points :  180  ->  100
points :  145  ->  99
points :  125  ->  101
points :  126  ->  100
points :  190  ->  96
points :  122  ->  98
points :  119  ->  101
points :  119  ->  101
points :  103  ->  101
points :  128  ->  100
points :  190  ->  96
points :  300  ->  100
points :  193  ->  100
points :  129  ->  100
points :  128  ->  100
points :  132  ->  100
points :  134  ->  100
points :  140  ->  100
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 
Score:  19.5

Number of candidates:  31
points :  119  ->  101
points :  157  ->  100
points :  122  ->  98
points :  103  ->  101
points :  167  ->  100
points :  118  ->  102
points :  152  ->  100
points :  112  ->  100
points :  143  ->  100
points :  133  ->  99
points :  138  ->  100
points :  236  ->  100
points :  162  ->  100
0 1 2 3 4 5 6 icp: Insufficient destination points
7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 
Score:  28.0

Number of candidates:  28
points :  101  ->  101
points :  124  ->  100
points :  115  ->  96
points :  111  ->  100
points :  107  ->  101
points :  108  ->  100
points :  193  ->  100
points :  123  ->  101
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 
Score:  28.0

Number of candidates:  18
points :  145  ->  99
points :  138  ->  100
points :  138  ->  100
points :  139  ->  101
points :  138  ->  100
points :  138  ->  100
points :  136  ->  100
points :  219  ->  100
points :  207  ->  100
points :  140  ->  100
points :  131  ->  100
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 
Score:  25.0

Number of candidates:  20
points :  107  ->  101
points :  111  ->  100
points :  111  ->  100
points :  101  ->  101
points :  102  ->  100
points :  113  ->  99
points :  104  ->  100
points :  107  ->  101
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 
Score:  26.0

Number of candidates:  20
points :  149  ->  100
points :  145  ->  99
points :  120  ->  100
points :  184  ->  100
points :  136  ->  100
points :  142  ->  100
points :  167  ->  100
points :  163  ->  100
points :  174  ->  100
points :  167  ->  100
points :  113  ->  99
points :  103  ->  101
points :  130  ->  97
points :  115  ->  96
points :  160  ->  100
points :  157  ->  100
points :  123  ->  101
points :  142  ->  100
points :  158  ->  101
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 
Score:  26.0

点数文字の認識は、特に悪くなってはいなさそうです。これでいけそう。
間引き後の輪郭点数は、ときどき100点から少し離れています。小数の丸めと比較のあたりの問題化とは思いますが、まあこれで進めます。

デバッグ表示を消して、改めて処理時間測定します。

class contour_dataset:
    def __init__(self, ctr):
        self.ctr = ctr.copy()
        self.rrect = cv2.minAreaRect(ctr)
        self.box = cv2.boxPoints(self.rrect)
        self.solid = create_solid_contour(ctr)
        n = 100
        if n >= ctr.shape[0]:
            self.pts = np.array([p for p in ctr[:,0,:]])
        else:            
            r = n / ctr.shape[0]
            self.pts = np.zeros((100,2), 'int')
            pts = []
            for i in range(ctr.shape[0]):
                f = math.modf(i*r)[0] 
                if (f <= r/2) or (f > 1.0 - r/2):
                    pts += [ctr[i,0,:]]
            self.pts = np.array(pts)

ts3 = []
for img, sel in zip(imgs, templates_sel):
    if sel == 0:
        templates = templates2019
    elif sel == 1:
        templates = templates2020
    else:
        templates = templates2021
    ts3 += [test_harupan_timeit(img, templates, svm)]

Number of candidates:  22
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 
Number of candidates:  22
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 
Number of candidates:  22
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 
Number of candidates:  22
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 
Number of candidates:  22
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 
Number of candidates:  22
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 
Score:  26.0
Average process time:  3.411894039999993

Number of candidates:  23
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 
Number of candidates:  23
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 
Number of candidates:  23
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 
Number of candidates:  23
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 
Number of candidates:  23
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 
Number of candidates:  23
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 
Score:  26.5
Average process time:  3.6053899799999956

Number of candidates:  23
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 
Number of candidates:  23
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 
Number of candidates:  23
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 
Number of candidates:  23
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 
Number of candidates:  23
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 
Number of candidates:  23
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 
Score:  25.0
Average process time:  1.5578179599999884

Number of candidates:  25
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 
Number of candidates:  25
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 
Number of candidates:  25
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 
Number of candidates:  25
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 
Number of candidates:  25
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 
Number of candidates:  25
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 
Score:  19.5
Average process time:  1.460823019999998

Number of candidates:  31
0 1 2 3 4 5 6 icp: Insufficient destination points
7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 
Number of candidates:  31
0 1 2 3 4 5 6 icp: Insufficient destination points
7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 
Number of candidates:  31
0 1 2 3 4 5 6 icp: Insufficient destination points
7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 
Number of candidates:  31
0 1 2 3 4 5 6 icp: Insufficient destination points
7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 
Number of candidates:  31
0 1 2 3 4 5 6 icp: Insufficient destination points
7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 
Number of candidates:  31
0 1 2 3 4 5 6 icp: Insufficient destination points
7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 
Score:  28.0
Average process time:  2.703369799999996

Number of candidates:  28
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 
Number of candidates:  28
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 
Number of candidates:  28
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 
Number of candidates:  28
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 
Number of candidates:  28
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 
Number of candidates:  28
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 
Score:  28.0
Average process time:  2.9763017799999942

Number of candidates:  18
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 
Number of candidates:  18
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 
Number of candidates:  18
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 
Number of candidates:  18
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 
Number of candidates:  18
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 
Number of candidates:  18
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 
Score:  25.0
Average process time:  1.883919160000005

Number of candidates:  20
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 
Number of candidates:  20
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 
Number of candidates:  20
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 
Number of candidates:  20
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 
Number of candidates:  20
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 
Number of candidates:  20
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 
Score:  26.0
Average process time:  2.917304880000006

Number of candidates:  20
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 
Number of candidates:  20
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 
Number of candidates:  20
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 
Number of candidates:  20
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 
Number of candidates:  20
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 
Number of candidates:  20
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 
Score:  26.0
Average process time:  2.4312215799999874

for t1,t2,t3 in zip(ts, ts2, ts3):
    print('{:.3f} vs '.format(t1), '{:.3f} vs'.format(t2), '{:.3f}'.format(t3))

8.822 vs  7.391 vs 3.412
8.280 vs  6.903 vs 3.605
6.180 vs  3.850 vs 1.558
5.870 vs  2.600 vs 1.461
4.848 vs  3.601 vs 2.703
4.439 vs  3.107 vs 2.976
4.840 vs  3.779 vs 1.884
3.853 vs  3.112 vs 2.917
6.385 vs  5.484 vs 2.431

概ね速くなりました。
元々輪郭点数が少なかったものはそれほど変わらず。

人間の体感的には少し待ちが出そうですが、一応リアルタイム処理できるかな？

以上

今度こそ調整は終了にしようかと。
気が変わらなければ、次回はPCのカメラ画像からのリアルタイム処理の作成になります。

OpenCVやってみる - 41. 処理の調整3

春のパン祭りシールの点数文字認識処理は、これ以上調整はなしにしようと思っていましたが、もう少しがんばってみます。

方針

前回の結果で、問題点の1つは

• シールの白抜き部分(点数文字が含まれる)の間隔が小さく、輪郭がつながってしまう

ということだったかと。
画像の2値化処理のところに戻って、うまく調整できないかと考えました。

2値化の調整

今作っている処理では、色相(Hue)、彩度(Saturation)を使って、点数シールのピンク領域を前景領域とした2値化画像を作っています。
今まで使った7つの画像と、前回うまくいかなかった画像でいくつか試してみます。

一旦まずはライブラリ等の下準備を。

import cv2
import numpy as np
%matplotlib inline
from matplotlib import pyplot as plt
import math
import copy
import random

img1 = cv2.imread('harupan_190428_1.jpg')
img2 = cv2.imread('harupan_190428_2.jpg')
img3 = cv2.imread('harupan_200317_1.jpg')
img4 = cv2.imread('harupan_210227_2.jpg')
img5 = cv2.imread('harupan_210402_1.jpg')
img6 = cv2.imread('harupan_210402_2.jpg')
img7 = cv2.imread('harupan_210414_1.jpg')
img8 = cv2.imread('harupan_220330_1.jpg')
img9 = cv2.imread('harupan_220330_2.jpg')

今まで使っていた処理から、2値化の部分だけ抜き出します。
また、彩度情報も2値化に使用していますが、これも調整が必要かと思い、閾値を設定できるようにしました。

def harupan_binarize(image, res_th=800, sat_th=100):
    h, w, chs = image.shape
    if h > res_th or w > res_th:
        k = float(res_th)/h if w > h else float(res_th)/w
    else:
        k = 1.0
    img = cv2.resize(image, None, fx=k, fy=k, interpolation=cv2.INTER_AREA)
    hsv = cv2.cvtColor(img, cv2.COLOR_BGR2HSV)
    # Convert hue value (rotation, mask by saturation)
    hsv[:,:,0] = np.where(hsv[:,:,0] < 50, hsv[:,:,0]+180, hsv[:,:,0])
    hsv[:,:,0] = np.where(hsv[:,:,1] < sat_th, 0, hsv[:,:,0])
    # Thresholding with cv2.inRange()
    binary_img = cv2.inRange(hsv[:,:,0], 135, 190)
    return binary_img

試したこと

• 膨張処理の実施
  2値化した後、膨張処理によってピンク領域を広げる→点数文字領域を縮小する、ということによって、輪郭どうしがつながるのを抑制する
• 彩度情報でピンク領域かどうかのふるい分けをしているが、この条件を緩和してピンク領域を増やすことで、膨張処理と似た効果を得る
  • 最初に画像のリサイズ(縮小)を行っているが、cv2.INTER_AREAを使っているので、ピンク色が隣接ピクセルまでにじんでくる
  • カメラ側での画像処理(ディベイヤー処理など？)で同じくピンク色が近くのピクセルまでにじむ？

ということを試します。

各画像5つを並べていますが、

• 元画像
• 今まで通りの2値化
• 彩度閾値緩和
• 膨張処理
• 彩度閾値緩和 + 膨張処理

となっています。

imgs = [img1, img2, img3, img4, img5, img6, img7, img8, img9]
binimgs = []

for img in imgs:
    binimg = harupan_binarize(img)
    binimg_sat = harupan_binarize(img, sat_th=50)
    kernel = np.ones((3,3), np.uint8)
    binimg_dil = cv2.dilate(binimg, kernel, iterations=1)
    binimg_sat_dil = cv2.dilate(binimg_sat, kernel, iterations=1)
    plt.figure(figsize=(25.6,9.6), dpi=100)
    plt.subplot(1,5,1), plt.imshow(cv2.cvtColor(img, cv2.COLOR_BGR2RGB), interpolation='none'), plt.xticks([]), plt.yticks([])
    plt.subplot(1,5,2), plt.imshow(binimg, cmap='gray', interpolation='none'), plt.xticks([]), plt.yticks([])
    plt.subplot(1,5,3), plt.imshow(binimg_sat, cmap='gray', interpolation='none'), plt.xticks([]), plt.yticks([])
    plt.subplot(1,5,4), plt.imshow(binimg_dil, cmap='gray', interpolation='none'), plt.xticks([]), plt.yticks([])
    plt.subplot(1,5,5), plt.imshow(binimg_sat_dil, cmap='gray', interpolation='none'), plt.xticks([]), plt.yticks([])
    plt.show()

ここでは、plt.imshow()のinterpolation引数で'none'を指定してみました。これによって、アンチエイリアス処理等入らない、生のピクセル配列が見やすくなりました。

https://bftnagoya.hateblo.jp/entry/2021/04/06/164423

plt.figure()でのグラフサイズ設定にもよって見た目が変わったので、実はこれもピクセル配列がそのまま見えているわけではないかも。

処理の変更で、2値化の結果にそれなりに違いは出たかな。

• 彩度閾値の変更、膨張処理のいずれでもシール内部の黒領域が減っている。
• 画像により、どちらのほうが効果が高いか異なる。
  • 今までの画像では、膨張処理のほうが効いている。
  • 前回の低解像度画像では、彩度閾値変更のほうが効いている。
• 両方適用すると点数文字が消えそうになっている…やり過ぎか。

今までの画像では特に文字認識に問題はなかったので、低解像度画像で効果のありそうな、彩度閾値変更のほうを試してみたいと思います。

スクリプト変更

シール点数計算スクリプト(harupan.py)で、detect_candidate_contours()関数(画像から点数輪郭を取得する)を以下のように変更します。

######################################################
# Detecting contours
######################################################
def reduce_resolution(img, res_th=800):
    h, w, chs = img.shape
    if h > res_th or w > res_th:
        k = float(res_th)/h if w > h else float(res_th)/w
    else:
        k = 1.0
    rtn_img = cv2.resize(img, None, fx=k, fy=k, interpolation=cv2.INTER_AREA)
    return rtn_img

def harupan_binarize(img, sat_th=100):
    hsv = cv2.cvtColor(img, cv2.COLOR_BGR2HSV)
    # Convert hue value (rotation, mask by saturation)
    hsv[:,:,0] = np.where(hsv[:,:,0] < 50, hsv[:,:,0]+180, hsv[:,:,0])
    hsv[:,:,0] = np.where(hsv[:,:,1] < sat_th, 0, hsv[:,:,0])
    # Thresholding with cv2.inRange()
    binary_img = cv2.inRange(hsv[:,:,0], 135, 190)
    return binary_img

def detect_candidate_contours(image, res_th=800, sat_th=100):
    img = reduce_resolution(image, res_th)
    binimg = harupan_binarize(img, sat_th)
    # Retrieve all points on the contours (cv2.CHAIN_APPROX_NONE)
    contours, hierarchy = cv2.findContours(binimg, cv2.RETR_TREE, cv2.CHAIN_APPROX_NONE)
    indices0 = [i for i,hier in enumerate(hierarchy[0,:,:]) if hier[3] == -1]
    indices1 = [i for i,hier in enumerate(hierarchy[0,:,:]) if hier[3] in indices0]
    contours1 = [contours[i] for i in indices1]
    contours1_filtered = [ctr for ctr in contours1 if cv2.contourArea(ctr) > float(res_th)*float(res_th)/4000]
    return contours1_filtered, img

変更したスクリプトを読み込んでおきます。

from harupan_data.harupan import *

テンプレート再作成、SVM再学習

輪郭検出処理も変わったので、テンプレートの作成とSVM学習を改めてやり直します。

輪郭検出

まずは各画像で輪郭検出をしておきます。

resized_imgs = []
ctrs_all = []
original_img_idx = []
subctrs_all = []
subimgs_all = []

plt.figure(figsize=(25.6, 9.6), dpi=200)
for i,img in enumerate(imgs):
    ctrs, im = detect_candidate_contours(img, sat_th=50)
    resized_imgs += [im]
    ctrs_all += [ctrs]

    for ctr in ctrs:
        original_img_idx += [i]
        subimg, subctr = create_contour_area_image(im, ctr)
        subctrs_all += [subctr]
        subimgs_all += [subimg]

    ctrs_img = cv2.drawContours(im.copy(), ctrs, -1, (0,255,0), 3)
    plt.subplot(2,5,1+i), plt.imshow(cv2.cvtColor(ctrs_img, cv2.COLOR_BGR2RGB)), plt.xticks([]), plt.yticks([])
plt.show()

ひとまず全画像必要な点数文字の輪郭は取得できたよう。

データセットも用意しておきます。

ctr_datasets_all = [contour_dataset(ctr) for ctr in subctrs_all]

テンプレート選択、正解ラベル作成

今までと同様、1つ目、3つ目、5つ目の画像から点数文字テンプレートを選びます。
また、学習用データ向けに、正解ラベルも用意します。

from ipywidgets import interact, fixed

def draw_contour(img, ctrs, idx):
    img_with_ctr = cv2.drawContours(img.copy(), [ctrs[idx]], -1, (0,255,0), 2)
    plt.figure(figsize=(6.4,4.8), dpi=100)
    plt.imshow(cv2.cvtColor(img_with_ctr, cv2.COLOR_BGR2RGB)), plt.xticks([]), plt.yticks([])
    plt.show()

def draw_contour_interact(i_img, idx):
    draw_contour(resized_imgs[i_img], ctrs_all[i_img], idx)

interact(draw_contour_interact, i_img=fixed(0), idx=(0, len(ctrs_all[0])-1));

labels1 = [
    -1, -1, -1, -1, -1,
    -1, 5, 0, 5, 1,
    5, 0, 2, 1, 2,
    -1, 1, 1, 5, 0,
    2, 5, 0, 2, 5,
    0, 1, 2, 5, 0,
    1, 3, 2, 1, 5,
    0
]

template_idx1 = {0:11, 1:13, 2:20, 3:31, 5:8}

interact(draw_contour_interact, i_img=fixed(1), idx=(0, len(ctrs_all[1])-1));

labels2 = [
    -1, -1, -1, -1, 5,
    0, 5, 1, 5, 0,
    2, 1, 2, -1, -1, 
    1, 1, 5, 0, 2,
    5, 0, 2, 5, 0,
    1, 2, 5, 0, 1,
    2, 3, 1, 5, 0
]

interact(draw_contour_interact, i_img=fixed(2), idx=(0, len(ctrs_all[2])-1));

labels3 = [
    -1, -1, -1, -1, -1,
    -1, -1, -1, -1, -1,
    1, 1, 5, 0, 1,
    1, 5, 0, 5, 0,
    2, -1, -1, -1, 1,
    1, 1, 1, 1, 1,
    1, 2, 1, 2, 1,
    2, 5, 0, 1, 1
]

template_idx3 = {0:13, 1:14, 2:31, 5:18}

interact(draw_contour_interact, i_img=fixed(3), idx=(0, len(ctrs_all[3])-1));

labels4 = [
    -1, -1, -1, -1, -1,
    5, 0, 2, 5, 0,
    1, 1, 2, 1, -1,
    -1, 1, -1, 1, 1,
    2, 2, -1, -1, -1,
    -1, -1, -1, 1, 1,
    1, 1, -1, 1, -1
]

interact(draw_contour_interact, i_img=fixed(4), idx=(0, len(ctrs_all[4])-1));

labels5 = [
    -1, -1, 0, 1, 5,
    2, -1, 1, 1, -1,
    5, 1, 0, -1, 5,
    2, 0, 1, 5, 0,
    1, 1, 2, 1, 1,
    1, 1, 2, 2, -1,
    -1, -1, -1, 1, 1,
    1, 1, -1, 1
]

template_idx5 = {0:12, 1:11, 2:5, 5:10}

interact(draw_contour_interact, i_img=fixed(5), idx=(0, len(ctrs_all[5])-1));

labels6 = [
    -1, 0, 1, 5, 2,
    -1, 1, 1, 5, 1,
    0, 5, 0, 2, 1,
    5, 0, 2, 1, 1,
    2, 2, -1, 1, -1,
    -1, 1, 1, 1, 1,
    1, 1, 1, 1
]

interact(draw_contour_interact, i_img=fixed(6), idx=(0, len(ctrs_all[6])-1));

labels7 = [
    -1, 1, 2, 2, 2,
    2, -1, -1, 1, 2,
    2, 2, 1, -1, -1,
    -1, 2, 1, 2, 1,
    1, -1
]

PCカメラ画像でも確認します。

interact(draw_contour_interact, i_img=fixed(7), idx=(0, len(ctrs_all[7])-1));

labels8 = [
    1, 2, 2, 2, 2,
    5, 1, 2, 5, 5,
    0, 0, 5, 2, 1,
    1, 2, 5, 0, 2,
    5, 0, 2, 1, -1
]

interact(draw_contour_interact, i_img=fixed(8), idx=(0, len(ctrs_all[8])-1));

labels9 = [
    5, 0, 5, 0, 1,
    2, 5, 1, 5, 0,
    2, 2, 2, 2, 1,
    1, 5, 0, 1, 2,
    2, 5, 2, 2
]

labels_all = labels1 + labels2 + labels3 + labels4 + labels5 + labels6 + labels7 + labels8 + labels9

テンプレート確認

subimgs1 = []
subctrs1 = []
binimgs1 = []
subctrs1_selected_pts = []
for i,(num, idx) in enumerate(template_idx1.items()):
    img, ctr = create_contour_area_image(resized_imgs[0], ctrs_all[0][idx])
    binimg, M, ctr2 = create_upright_solid_contour(ctr)
    img2 = cv2.warpAffine(img.copy(), M, (binimg.shape[1], binimg.shape[0]))
    subimgs1 += [img2]
    subctrs1 += [ctr2]
    binimgs1 += [binimg]
    ctr_selected_pts = [j for j in range(ctr2.shape[0]) if j % 5 == 0]
    if i != 0:
        subctrs1_selected_pts += [ctr_selected_pts]
    ctr_img = cv2.drawContours(img2.copy(), [ctr2], -1, (0,255,0), 2)
    pts_img = img2.copy()
    for p in ctr_selected_pts:
        pts_img = cv2.drawMarker(pts_img, ctr2[p,0,:], (0,255,0), markerType=cv2.MARKER_CROSS, markerSize=3)
    plt.subplot(3,5,1+i), plt.imshow(cv2.cvtColor(ctr_img, cv2.COLOR_BGR2RGB)), plt.xticks([]), plt.yticks([])
    plt.subplot(3,5,6+i), plt.imshow(binimg,cmap='gray'), plt.xticks([]), plt.yticks([])
    plt.subplot(3,5,11+i), plt.imshow(cv2.cvtColor(pts_img, cv2.COLOR_BGR2RGB), cmap='gray'), plt.xticks([]), plt.yticks([])
plt.show()

subimgs3 = []
subctrs3 = []
binimgs3 = []
subctrs3_selected_pts = []
for i,(num, idx) in enumerate(template_idx3.items()):
    img, ctr = create_contour_area_image(resized_imgs[2], ctrs_all[2][idx])
    binimg, M, ctr2 = create_upright_solid_contour(ctr)
    img2 = cv2.warpAffine(img.copy(), M, (binimg.shape[1], binimg.shape[0]))
    subimgs3 += [img2]
    subctrs3 += [ctr2]
    binimgs3 += [binimg]
    ctr_selected_pts = [j for j in range(ctr2.shape[0]) if j % 5 == 0]
    if i != 0:
        subctrs3_selected_pts += [ctr_selected_pts]
    ctr_img = cv2.drawContours(img2.copy(), [ctr2], -1, (0,255,0), 2)
    pts_img = img2.copy()
    for p in ctr_selected_pts:
        pts_img = cv2.drawMarker(pts_img, ctr2[p,0,:], (0,255,0), markerType=cv2.MARKER_CROSS, markerSize=3)
    plt.subplot(3,5,1+i), plt.imshow(cv2.cvtColor(ctr_img, cv2.COLOR_BGR2RGB)), plt.xticks([]), plt.yticks([])
    plt.subplot(3,5,6+i), plt.imshow(binimg,cmap='gray'), plt.xticks([]), plt.yticks([])
    plt.subplot(3,5,11+i), plt.imshow(cv2.cvtColor(pts_img, cv2.COLOR_BGR2RGB), cmap='gray'), plt.xticks([]), plt.yticks([])
plt.show()

subimgs3.insert(3, subimgs1[3])
subctrs3.insert(3, subctrs1[3])
binimgs3.insert(3, binimgs1[3])
subctrs3_selected_pts.insert(2, subctrs1_selected_pts[2])

subimgs5 = []
subctrs5 = []
binimgs5 = []
subctrs5_selected_pts = []
for i,(num, idx) in enumerate(template_idx5.items()):
    img, ctr = create_contour_area_image(resized_imgs[4], ctrs_all[4][idx])
    binimg, M, ctr2 = create_upright_solid_contour(ctr)
    img2 = cv2.warpAffine(img.copy(), M, (binimg.shape[1], binimg.shape[0]))
    subimgs5 += [img2]
    subctrs5 += [ctr2]
    binimgs5 += [binimg]
    ctr_selected_pts = [j for j in range(ctr2.shape[0]) if j % 5 == 0]
    if i != 0:
        subctrs5_selected_pts += [ctr_selected_pts]
    ctr_img = cv2.drawContours(img2.copy(), [ctr2], -1, (0,255,0), 2)
    pts_img = img2.copy()
    for p in ctr_selected_pts:
        pts_img = cv2.drawMarker(pts_img, ctr2[p,0,:], (0,255,0), markerType=cv2.MARKER_CROSS, markerSize=3)
    plt.subplot(3,5,1+i), plt.imshow(cv2.cvtColor(ctr_img, cv2.COLOR_BGR2RGB)), plt.xticks([]), plt.yticks([])
    plt.subplot(3,5,6+i), plt.imshow(binimg,cmap='gray'), plt.xticks([]), plt.yticks([])
    plt.subplot(3,5,11+i), plt.imshow(cv2.cvtColor(pts_img, cv2.COLOR_BGR2RGB), cmap='gray'), plt.xticks([]), plt.yticks([])
plt.show()

subimgs5.insert(3, subimgs1[3])
subctrs5.insert(3, subctrs1[3])
binimgs5.insert(3, binimgs1[3])
subctrs5_selected_pts.insert(2, subctrs1_selected_pts[2])

テンプレートデータセットの作成

# Prepare template data for "0"
templates1 = [template_dataset(subctrs1[0], 0)]
templates3 = [template_dataset(subctrs3[0], 0)]
templates5 = [template_dataset(subctrs5[0], 0)]
# Prepare template data for other numbers
numbers = [1, 2, 3, 5]
for i,num in enumerate(numbers):
    templates1 += [template_dataset(subctrs1[i+1], num, subctrs1_selected_pts[i])]
    templates3 += [template_dataset(subctrs3[i+1], num, subctrs3_selected_pts[i])]
    templates5 += [template_dataset(subctrs5[i+1], num, subctrs5_selected_pts[i])]

一致度計算

templates_sel = [1,1,3,5,5,5,5,5,5]
def select_template(i):
    img_idx = original_img_idx[i]
    if templates_sel[img_idx] == 1:
        return templates1
    elif templates_sel[img_idx] == 3:
        return templates3
    elif templates_sel[img_idx] == 5:
        return templates5
    else:
        return templates1

similarities_all = []
converted_imgs_all = []
print('  Contour No. ', end='')
for i,target_ctr in enumerate(ctr_datasets_all):
    templates = select_template(i)
    print(i, ' ', end='')
    sims, imgs = get_similarities(target_ctr, templates)
    similarities_all += [sims]
    converted_imgs_all += [imgs]

  Contour No. 0  1  2  3  4  5  6  7  8  9  10  11  12  13  14  15  16  17  18  19  20  21  22  23  24  25  26  27  28  29  30  31  32  33  34  35  36  37  38  39  40  41  42  43  44  45  46  47  48  49  50  51  52  53  54  55  56  57  58  59  60  61  62  63  64  65  66  67  68  69  70  71  72  73  74  75  76  77  78  79  80  81  82  83  84  85  86  87  88  89  90  91  92  93  94  95  96  97  98  99  100  101  102  103  104  105  106  107  108  109  110  111  112  113  114  115  116  117  118  119  120  121  122  123  124  125  126  127  128  129  130  131  132  133  134  135  136  137  138  139  140  141  142  143  144  145  146  147  148  149  150  151  152  153  154  155  156  157  158  159  160  161  162  163  164  165  166  167  168  169  170  171  172  173  174  175  176  177  178  179  180  181  182  183  184  185  186  187  188  189  190  191  192  193  194  195  196  197  198  199  200  201  202  203  204  205  206  207  208  209  210  211  212  213  214  215  216  217  218  219  220  221  222  223  224  225  226  227  228  229  230  231  232  233  234  235  236  237  238  239  240  241  242  243  244  245  246  247  248  249  250  251  252  253  254  255  256  257  258  259  260  261  262  263  264  265  266  267  268  269  270  271  272  273  274  275  276  277  278  279  280  281  282  283  284  285  286  287  288  289  

SVM学習

前にまとめた関連処理を改めて示します。

def get_random_sample(data_in, labels_in, selected_labels, n_samples, seed=None):
    random.seed(seed)
    data_rtn = []
    labels_rtn = []
    for lab in selected_labels:
        samples = [d for i,d in enumerate(data_in) if labels_in[i]==lab]
        n = min(n_samples, len(samples))
        data_rtn += random.sample(samples, n)
        labels_rtn += [lab] * n
    return data_rtn, labels_rtn

def prepare_svm(train_data, train_labels):
    svm = cv2.ml.SVM_create()
    svm.setKernel(cv2.ml.SVM_LINEAR)
    svm.setType(cv2.ml.SVM_C_SVC)
    svm.setC(100)
    svm.setGamma(1)
    svm.train(np.array(train_data, 'float32'), cv2.ml.ROW_SAMPLE, np.array(train_labels))
    return svm

def print_stat(svm_results, svm_labels):
    stats = {k:{k2:0 for k2 in [-1, 0, 1, 2, 3, 5]} for k in [-1, 0, 1, 2, 3, 5]}
    for res, lab in zip(svm_results[1], svm_labels):
        stats[lab][int(res[0])] += 1
    for k,v in stats.items():
        print('label {:>2}'.format(k), ': {', end='')
        for k2,v2 in v.items():
            print('{}: {:>2}, '.format(k2,v2), end='')
        print('}')

def print_similarity_vector(sim, end=''):
    print('[',end='')
    for s in sim: print('{:.3f}, '.format(s), end='')
    print(']', end=end)

svm_inputs = copy.deepcopy(similarities_all)
svm_labels = copy.deepcopy(labels_all)

train_data, train_labels = get_random_sample(svm_inputs, svm_labels, [-1,0,1,2,3,5], 20, seed=1)
svm = prepare_svm(train_data, train_labels)

SVM推論実施

result = svm.predict(np.array(svm_inputs, 'float32'))
print_stat(result, svm_labels)

label -1 : {-1: 55, 0:  4, 1:  5, 2:  0, 3:  0, 5:  1, }
label  0 : {-1:  2, 0: 34, 1:  0, 2:  0, 3:  0, 5:  0, }
label  1 : {-1:  1, 0:  0, 1: 86, 2:  1, 3:  0, 5:  0, }
label  2 : {-1:  0, 0:  0, 1:  0, 2: 57, 3:  0, 5:  0, }
label  3 : {-1:  0, 0:  0, 1:  0, 2:  0, 3:  2, 5:  0, }
label  5 : {-1:  2, 0:  0, 1:  0, 2:  0, 3:  0, 5: 40, }

ちょこちょこ失敗してる…
どこで失敗したか確認します。

subimgs = copy.deepcopy(subimgs_all)
subctrs = copy.deepcopy(subctrs_all)

for i,(sims,lab,res,img,ctr) in enumerate(zip(svm_inputs, svm_labels, result[1], subimgs, subctrs)):
    if lab != res[0]:
        print('No.', i)
        print('{: }'.format(lab), ' -> ', '{: d}'.format(int(res[0])), ' [',end='')
        for s in sims: print('{:.3f}, '.format(s), end='');
        print(']')
        img2 = cv2.drawContours(img.copy(), [ctr], -1, (0,255,0), 1)
        plt.imshow(cv2.cvtColor(img2, cv2.COLOR_BGR2RGB)),plt.xticks([]),plt.yticks([])
        plt.show()

No. 2
-1  ->   1  [0.662, 0.825, 0.750, 0.728, 0.772, ]

No. 36
-1  ->   0  [0.823, 0.853, 0.660, 0.641, 0.697, ]

No. 50
-1  ->   5  [0.678, 0.659, 0.641, 0.769, 0.792, ]

No. 88
 0  ->  -1  [0.845, 0.712, 0.670, 0.654, 0.699, ]

No. 111
-1  ->   1  [0.577, 0.877, 0.797, 0.779, 0.815, ]

No. 113
-1  ->   0  [0.907, 0.868, 0.664, 0.685, 0.733, ]

No. 128
-1  ->   1  [0.563, 0.786, 0.790, 0.756, 0.765, ]

No. 136
-1  ->   1  [0.629, 0.826, 0.818, 0.755, 0.801, ]

No. 146
-1  ->   0  [0.821, 0.906, 0.668, 0.638, 0.671, ]

No. 155
-1  ->   0  [0.906, 0.774, 0.646, 0.698, 0.700, ]

No. 195
 0  ->  -1  [0.803, 0.755, 0.731, 0.611, 0.716, ]

No. 212
 1  ->  -1  [0.776, 0.693, 0.722, 0.682, 0.669, ]

No. 215
 1  ->   2  [0.572, 0.964, 0.847, 0.802, 0.640, ]

No. 225
-1  ->   1  [0.563, 0.901, 0.847, 0.799, 0.719, ]

No. 258
 5  ->  -1  [0.659, 0.723, 0.684, 0.790, 0.668, ]

No. 282
 5  ->  -1  [0.514, 0.643, 0.647, 0.787, 0.619, ]

• 212番の輪郭は、初期一致度が0.7に達しなかったので、ICPが実施されず、"1"への一致度が高まらなかったのが問題か。
• 215番の輪郭は、なぜこれでうまくいかないのか…
• 最後の2つの"5"は、よく見ると輪郭が結構歪んでる。シール領域を広げるような閾値変更を入れたので、点数文字領域が必要以上に浸食されたのか。
  閾値の調整が必要？それとも画像全体の明るさ調整が必要？
  ただ、そうしてしまうと、元の問題(輪郭どうしがくっ付いてしまう)がまた発生するか…

OpenCVでのRGB→HSV変換の式を見ると、色相(Hue)、彩度(Saturation)は明るさには依存しなさそう。

https://docs.opencv.org/4.5.1/de/d25/imgproc_color_conversions.html

やっぱり彩度の閾値検討か…

彩度閾値検討

最後の2つの"5"の画像を改めて見てみます。

plt.subplot(121), plt.imshow(cv2.cvtColor(subimgs[258], cv2.COLOR_BGR2RGB)),plt.xticks([]),plt.yticks([])
plt.subplot(122), plt.imshow(cv2.cvtColor(subimgs[282], cv2.COLOR_BGR2RGB)),plt.xticks([]),plt.yticks([])
plt.show()

"5"の文字の内側にうっすら色がついているような。
彩度のヒストグラムを出してみる。

img258_hsv = cv2.cvtColor(subimgs[258], cv2.COLOR_BGR2HSV)
img282_hsv = cv2.cvtColor(subimgs[282], cv2.COLOR_BGR2HSV)
plt.subplot(121), plt.hist(img258_hsv[:,:,1].ravel(), 256, [0,256])
plt.subplot(122), plt.hist(img282_hsv[:,:,1].ravel(), 256, [0,256])
plt.show()

あまりはっきりした閾値が出ない…

間違えてRGB値で見てみましたが、どうもB、Gのほうがしっかり分かれていそう。

plt.subplot(121), plt.hist(subimgs[258][:,:,0].ravel(), 256, [0,256])
plt.subplot(122), plt.hist(subimgs[258][:,:,0].ravel(), 256, [0,256])
plt.show()
plt.subplot(121), plt.hist(subimgs[258][:,:,1].ravel(), 256, [0,256])
plt.subplot(122), plt.hist(subimgs[258][:,:,1].ravel(), 256, [0,256])
plt.show()
plt.subplot(121), plt.hist(subimgs[258][:,:,2].ravel(), 256, [0,256])
plt.subplot(122), plt.hist(subimgs[258][:,:,2].ravel(), 256, [0,256])
plt.show()

今回はシールの色がピンク色で、色相も青寄りな部分もあったので、Gの値で識別するのがよさそうに思います。

トラックバーで動かしながら見てみる。

def thresholding_img258(thresh):
    ret, img2 = cv2.threshold(subimgs[258][:,:,1], thresh, 255, cv2.THRESH_BINARY)
    plt.imshow(img2, cmap='gray'), plt.xticks([]), plt.yticks([])
    plt.show()

interact(thresholding_img258, thresh=(0,255));

• 閾値が58より下がると真っ黒になる(全値が閾値を超えると反転するのか？)
• 85ぐらいから"5"の文字の形が見え出す
• 128ぐらいがベストか
• 閾値を上げていくと文字形状が細くなっていき、最終的に真っ黒になる

大津の2値化検討

閾値をなんとなくで決めるのもあんまりかな、と思うので、大津の2値化を試してみる。
こちらは、二峰性の分布になっている前提で、最適な閾値を計算するアルゴリズムのよう。

https://docs.opencv.org/4.x/d7/d4d/tutorial_py_thresholding.html

ret258, img258_thresh = cv2.threshold(subimgs[258][:,:,1], 0, 255, cv2.THRESH_BINARY+cv2.THRESH_OTSU)
print('Otsu threshold: ', ret258)
plt.imshow(img258_thresh, cmap='gray'), plt.xticks([]), plt.yticks([])
plt.show()

Otsu threshold:  109.0

いい感じかな。
もう一つの画像でも。

ret282, img282_thresh = cv2.threshold(subimgs[282][:,:,1], 0, 255, cv2.THRESH_BINARY+cv2.THRESH_OTSU)
print('Otsu threshold: ', ret282)
plt.imshow(img282_thresh, cmap='gray'), plt.xticks([]), plt.yticks([])
plt.show()

Otsu threshold:  93.0

こちらもいい感じ。

処理の変更

処理は以下の流れを検討。

• まず今まで通り色相、彩度で点数文字領域を検出
• 点数文字領域の小画像(外接矩形による)を用意
• 小画像のG値で、大津の2値化を適用
  • 小画像を使っているので、この中に含まれるのはほぼシール領域のピンク色と白色だけになるかと。そうなればG値での2値化でOK。
• この小画像で改めて輪郭を検出

上の2値化画像で輪郭検出できるか？

ctrs, hier = cv2.findContours(img258_thresh, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_NONE)
print(len(ctrs))
print(ctrs[0].shape)
img = cv2.cvtColor(img258_thresh, cv2.COLOR_GRAY2RGB)
img = cv2.drawContours(img, ctrs, -1, (0,255,0), 1)
plt.imshow(img), plt.xticks([]), plt.yticks([])
plt.show()

1
(92, 1, 2)

ctrs, hier = cv2.findContours(img282_thresh, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_NONE)
print(len(ctrs))
print(ctrs[0].shape)
img = cv2.cvtColor(img282_thresh, cv2.COLOR_GRAY2RGB)
img = cv2.drawContours(img, ctrs, -1, (0,255,0), 1)
plt.imshow(img), plt.xticks([]), plt.yticks([])
plt.show()

1
(130, 1, 2)

一応輪郭は取得できました。2つ目の画像は下側はちょっと変な感じですが…

余計な輪郭領域も出てくる可能性がありそうなので、輪郭の周長を使って、必要なものか判定するようにします。
これで全画像で改めてやってみる。

スクリプトでは、refine_contours()関数ということで、大津の2値化により輪郭データを再生成する処理を作り、detect_candidate_contours()関数で輪郭検出を行った後にこれを実施します。

一旦ここではrefine_contours()関数を定義してこれを使って、輪郭検出～SVM学習、推論までを再度実施します。

この処理の中で、2値化画像そのままで輪郭検出を行うと、前景領域が画像の端に接触しているせいか、外接矩形の辺を含んだ輪郭が検出されてしまうことがありました。そのため、輪郭検出の前に、cv2.copyMakeBorder()で外周に背景領域を追加しました。

# image: Entire image containing multiple contours
# contours: Contours contained in "image" (Retrieved by cv2.findContours(), the origin is same as "image")
def refine_contours(image, contours):
    subctrs = []
    subimgs = []
    binimgs = []
    thresholds = []
    n_ctrs = []
    for ctr in contours:
        img, _ = create_contour_area_image(image, ctr)
        # Thresholding using G value in BGR format
        thresh, binimg = cv2.threshold(img[:,:,1], 0, 255, cv2.THRESH_BINARY + cv2.THRESH_OTSU)
        # Add black region around thresholded image, to detect contours correctly
        binimg = cv2.copyMakeBorder(binimg, 2,2,2,2, cv2.BORDER_CONSTANT, 0)
        ctrs2, _ = cv2.findContours(binimg, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_NONE)
        max_len = 0
        for ctr2 in ctrs2:
            if max_len <= ctr2.shape[0]:
                max_ctr = ctr2
                max_len = ctr2.shape[0]
        subctrs += [max_ctr]
        subimgs += [img]
        binimgs += [binimg]
        thresholds += [thresh]
        n_ctrs += [len(ctrs2)]
    debug_info = (binimgs, thresholds, n_ctrs)
    return subctrs, subimgs, debug_info

subctrs_all = []

i = 1
plt.figure(figsize=(12.8, 9.6), dpi=100)
for ctrs, img in zip(ctrs_all, resized_imgs):
    subctrs, subimgs, dbginfo = refine_contours(img, ctrs)
    subctrs_all += subctrs
    for subctr, subimg, binimg, th, nc in zip(subctrs, subimgs, dbginfo[0], dbginfo[1], dbginfo[2]):
        im = cv2.cvtColor(binimg, cv2.COLOR_GRAY2RGB)
        im = cv2.drawContours(im, [subctr], -1, (0,255,0), 1)
        im2 = cv2.copyMakeBorder(subimg, 2,2,2,2, cv2.BORDER_CONSTANT, value=(255,255,0))
        im = np.hstack((im2, im))
        plt.subplot(1,5,i), plt.imshow(cv2.cvtColor(im, cv2.COLOR_BGR2RGB)), plt.xticks([]), plt.yticks([]), plt.title(str(nc) + ', ' + str(th))
        if i == 5:
            i = 1
            plt.show()
            plt.figure(figsize=(12.8, 9.6), dpi=100)
        else:
            i+=1
if i != 1:
    plt.show();

<Figure size 1280x960 with 0 Axes>

画像の明るさに従って2値化の閾値が変わっている様子も見られます。

ctr_datasets_all = [contour_dataset(ctr) for ctr in subctrs_all]

テンプレート再作成

一応テンプレートも作り直します。

# ctr: Should be output of create_contour_area_image() (Origin of points is the origin of bounding box)
def create_upright_solid_contour(ctr):
    ctr2 = ctr.copy()
    (cx,cy),(w,h),angle = cv2.minAreaRect(ctr2)
    print((cx,cy),(w,h),angle)
    M = cv2.getRotationMatrix2D((cx,cy), angle, 1)
    for i in range(ctr2.shape[0]):
        ctr2[i,0,:] = ( M @ np.array([ctr2[i,0,0], ctr2[i,0,1], 1]) ).astype('int')
    rect = cv2.boundingRect(ctr2)
    img = np.zeros((rect[3],rect[2]), 'uint8')
    ctr2 -= rect[0:2]
    M[:,2] -= rect[0:2]
    img = cv2.drawContours(img, [ctr2], -1, 255,-1)
    return img, M, ctr2

subimgs1 = []
subctrs1 = []
binimgs1 = []
subctrs1_selected_pts = []
for i,(num, idx) in enumerate(template_idx1.items()):
    binimg, M, ctr2 = create_upright_solid_contour(subctrs_all[idx])
    img2 = cv2.copyMakeBorder(subimgs_all[idx].copy(), 2,2,2,2, cv2.BORDER_CONSTANT, 0)
    img2 = cv2.warpAffine(img2, M, (binimg.shape[1], binimg.shape[0]))
    subimgs1 += [img2]
    subctrs1 += [ctr2]
    binimgs1 += [binimg]
    ctr_selected_pts = [j for j in range(ctr2.shape[0]) if j % 5 == 0]
    if i != 0:
        subctrs1_selected_pts += [ctr_selected_pts]
    ctr_img = cv2.drawContours(img2.copy(), [ctr2], -1, (0,255,0), 2)
    pts_img = img2.copy()
    for p in ctr_selected_pts:
        pts_img = cv2.drawMarker(pts_img, ctr2[p,0,:], (0,255,0), markerType=cv2.MARKER_CROSS, markerSize=3)
    plt.subplot(3,5,1+i), plt.imshow(cv2.cvtColor(ctr_img, cv2.COLOR_BGR2RGB)), plt.xticks([]), plt.yticks([])
    plt.subplot(3,5,6+i), plt.imshow(binimg,cmap='gray'), plt.xticks([]), plt.yticks([])
    plt.subplot(3,5,11+i), plt.imshow(cv2.cvtColor(pts_img, cv2.COLOR_BGR2RGB), cmap='gray'), plt.xticks([]), plt.yticks([])
plt.show()

(21.5, 27.5) (49.0, 37.0) 90.0
(13.5, 26.0) (48.0, 21.0) 90.0
(22.68317222595215, 26.331684112548828) (49.15483856201172, 38.308929443359375) 84.2894058227539
(23.0, 29.0) (54.0, 40.0) 90.0
(16.576923370361328, 18.115385055541992) (23.533935546875, 30.201885223388672) 11.309932708740234

subimgs3 = []
subctrs3 = []
binimgs3 = []
subctrs3_selected_pts = []
for i,(num, idx) in enumerate(template_idx3.items()):
    global_idx = idx +len(ctrs_all[0]) +len(ctrs_all[1])
    binimg, M, ctr2 = create_upright_solid_contour(subctrs_all[global_idx])
    img2 = cv2.copyMakeBorder(subimgs_all[global_idx].copy(), 2,2,2,2, cv2.BORDER_CONSTANT, 0)
    img2 = cv2.warpAffine(img2, M, (binimg.shape[1], binimg.shape[0]))
    subimgs3 += [img2]
    subctrs3 += [ctr2]
    binimgs3 += [binimg]
    ctr_selected_pts = [j for j in range(ctr2.shape[0]) if j % 5 == 0]
    if i != 0:
        subctrs3_selected_pts += [ctr_selected_pts]
    ctr_img = cv2.drawContours(img2.copy(), [ctr2], -1, (0,255,0), 2)
    pts_img = img2.copy()
    for p in ctr_selected_pts:
        pts_img = cv2.drawMarker(pts_img, ctr2[p,0,:], (0,255,0), markerType=cv2.MARKER_CROSS, markerSize=3)
    plt.subplot(3,5,1+i), plt.imshow(cv2.cvtColor(ctr_img, cv2.COLOR_BGR2RGB)), plt.xticks([]), plt.yticks([])
    plt.subplot(3,5,6+i), plt.imshow(binimg,cmap='gray'), plt.xticks([]), plt.yticks([])
    plt.subplot(3,5,11+i), plt.imshow(cv2.cvtColor(pts_img, cv2.COLOR_BGR2RGB), cmap='gray'), plt.xticks([]), plt.yticks([])
plt.show()

subimgs3.insert(3, subimgs1[3])
subctrs3.insert(3, subctrs1[3])
binimgs3.insert(3, binimgs1[3])
subctrs3_selected_pts.insert(2, subctrs1_selected_pts[2])

(20.0, 30.0) (54.0, 34.0) 90.0
(14.694175720214844, 29.631237030029297) (53.20975875854492, 22.366018295288086) 88.02507019042969
(23.0, 29.5) (53.0, 40.0) 90.0
(15.0, 23.0) (40.0, 26.0) 90.0

subimgs5 = []
subctrs5 = []
binimgs5 = []
subctrs5_selected_pts = []
for i,(num, idx) in enumerate(template_idx5.items()):
    global_idx = idx +len(ctrs_all[0]) +len(ctrs_all[1]) +len(ctrs_all[2]) +len(ctrs_all[3])
    binimg, M, ctr2 = create_upright_solid_contour(subctrs_all[global_idx])
    img2 = cv2.copyMakeBorder(subimgs_all[global_idx].copy(), 2,2,2,2, cv2.BORDER_CONSTANT, 0)
    img2 = cv2.warpAffine(img2, M, (binimg.shape[1], binimg.shape[0]))
    subimgs5 += [img2]
    subctrs5 += [ctr2]
    binimgs5 += [binimg]
    ctr_selected_pts = [j for j in range(ctr2.shape[0]) if j % 5 == 0]
    if i != 0:
        subctrs5_selected_pts += [ctr_selected_pts]
    ctr_img = cv2.drawContours(img2.copy(), [ctr2], -1, (0,255,0), 2)
    pts_img = img2.copy()
    for p in ctr_selected_pts:
        pts_img = cv2.drawMarker(pts_img, ctr2[p,0,:], (0,255,0), markerType=cv2.MARKER_CROSS, markerSize=3)
    plt.subplot(3,5,1+i), plt.imshow(cv2.cvtColor(ctr_img, cv2.COLOR_BGR2RGB)), plt.xticks([]), plt.yticks([])
    plt.subplot(3,5,6+i), plt.imshow(binimg,cmap='gray'), plt.xticks([]), plt.yticks([])
    plt.subplot(3,5,11+i), plt.imshow(cv2.cvtColor(pts_img, cv2.COLOR_BGR2RGB), cmap='gray'), plt.xticks([]), plt.yticks([])
plt.show()

subimgs5.insert(3, subimgs1[3])
subctrs5.insert(3, subctrs1[3])
binimgs5.insert(3, binimgs1[3])
subctrs5_selected_pts.insert(2, subctrs1_selected_pts[2])

(18.000001907348633, 21.999996185302734) (28.621667861938477, 40.24921798706055) 26.56505012512207
(13.478048324584961, 21.897563934326172) (37.43586730957031, 18.717931747436523) 77.90524291992188
(23.389907836914062, 20.866973876953125) (32.087181091308594, 36.876312255859375) 16.699243545532227
(19.799999237060547, 17.899995803833008) (21.01903533935547, 33.98823165893555) 26.56505012512207

いずれもきちんと文字の輪郭が取れました。

ついでに、cv2.minAreaRect()の結果も見てみましたが、回転角が90°となっているものがいくつかあり。この関数では条件によって回転角が0°, 90°, 180°, 270°に丸められて、矩形サイズも元画像のものがそのまま使われる、ということかと思われます。

テンプレートデータ作成

# Prepare template data for "0"
templates1 = [template_dataset(subctrs1[0], 0)]
templates3 = [template_dataset(subctrs3[0], 0)]
templates5 = [template_dataset(subctrs5[0], 0)]
# Prepare template data for other numbers
numbers = [1, 2, 3, 5]
for i,num in enumerate(numbers):
    templates1 += [template_dataset(subctrs1[i+1], num, subctrs1_selected_pts[i])]
    templates3 += [template_dataset(subctrs3[i+1], num, subctrs3_selected_pts[i])]
    templates5 += [template_dataset(subctrs5[i+1], num, subctrs5_selected_pts[i])]

一致度計算

similarities_all = []
converted_imgs_all = []
print('  Contour No. ', end='')
for i,target_ctr in enumerate(ctr_datasets_all):
    templates = select_template(i)
    print(i, ' ', end='')
    sims, imgs = get_similarities(target_ctr, templates)
    similarities_all += [sims]
    converted_imgs_all += [imgs]

  Contour No. 0  1  2  3  4  5  6  7  8  9  10  11  12  13  14  15  16  17  18  19  20  21  22  23  24  25  26  27  28  29  30  31  32  33  34  35  36  37  38  39  40  41  42  43  44  45  46  47  48  49  50  51  52  53  54  55  56  57  58  59  60  61  62  63  64  65  66  67  68  69  70  71  72  73  74  75  76  77  78  79  80  81  82  83  84  85  86  87  88  89  90  91  92  93  94  95  96  97  98  99  100  101  102  103  104  105  106  107  108  109  110  111  112  113  114  115  116  117  118  119  120  121  122  123  124  125  126  127  128  129  130  131  132  133  134  135  136  137  138  139  140  141  142  143  144  145  146  147  148  149  150  151  152  153  154  155  icp: Insufficient destination points
156  157  158  159  160  161  162  163  164  165  166  167  168  169  170  171  172  173  174  175  176  177  178  179  180  181  182  183  184  185  186  187  188  189  190  191  192  193  194  195  196  197  198  199  200  201  202  203  204  205  206  207  208  209  210  211  212  213  214  215  216  217  218  219  220  221  222  223  224  225  226  227  228  229  230  231  232  233  234  235  236  237  238  239  240  241  242  243  244  245  246  247  248  249  250  251  252  253  254  255  256  257  258  259  260  261  262  263  264  265  266  267  268  269  270  271  272  273  274  275  276  277  278  279  280  281  282  283  284  285  286  287  288  289  

SVM学習、推論

svm_inputs = copy.deepcopy(similarities_all)
svm_labels = copy.deepcopy(labels_all)

train_data, train_labels = get_random_sample(svm_inputs, svm_labels, [-1,0,1,2,3,5], 20, seed=1)
svm = prepare_svm(train_data, train_labels)

result = svm.predict(np.array(svm_inputs, 'float32'))
print_stat(result, svm_labels)

label -1 : {-1: 44, 0: 17, 1:  2, 2:  0, 3:  0, 5:  2, }
label  0 : {-1:  3, 0: 33, 1:  0, 2:  0, 3:  0, 5:  0, }
label  1 : {-1:  0, 0:  0, 1: 88, 2:  0, 3:  0, 5:  0, }
label  2 : {-1:  0, 0:  0, 1:  0, 2: 57, 3:  0, 5:  0, }
label  3 : {-1:  0, 0:  0, 1:  0, 2:  0, 3:  2, 5:  0, }
label  5 : {-1:  0, 0:  0, 1:  0, 2:  0, 3:  0, 5: 42, }

まだ誤認識はありますが、点数計算に支障が出る感じではないので、このまま進めます。

subimgs = copy.deepcopy(subimgs_all)
subctrs = copy.deepcopy(subctrs_all)

for i,(sims,lab,res,img,ctr) in enumerate(zip(svm_inputs, svm_labels, result[1], subimgs, subctrs)):
    if lab != res[0]:
        print('No.', i)
        print('{: }'.format(lab), ' -> ', '{: d}'.format(int(res[0])), ' [',end='')
        for s in sims: print('{:.3f}, '.format(s), end='');
        print(']')
        img2 = cv2.copyMakeBorder(img.copy(), 2,2,2,2, cv2.BORDER_CONSTANT, 0)
        img2 = cv2.drawContours(img2, [ctr], -1, (0,255,0), 1)
        plt.imshow(cv2.cvtColor(img2, cv2.COLOR_BGR2RGB)),plt.xticks([]),plt.yticks([])
        plt.show()

No. 0
-1  ->   0  [0.828, 0.849, 0.684, 0.649, 0.689, ]

No. 4
-1  ->   0  [0.828, 0.785, 0.686, 0.658, 0.688, ]

No. 5
-1  ->   0  [0.829, 0.793, 0.671, 0.622, 0.665, ]

No. 36
-1  ->   0  [0.831, 0.844, 0.685, 0.659, 0.704, ]

No. 39
-1  ->   0  [0.807, 0.778, 0.689, 0.645, 0.686, ]

No. 50
-1  ->   5  [0.473, 0.681, 0.601, 0.702, 0.801, ]

No. 71
-1  ->   0  [0.850, 0.682, 0.619, 0.638, 0.707, ]

No. 74
-1  ->   0  [0.844, 0.680, 0.624, 0.637, 0.732, ]

No. 75
-1  ->   0  [0.837, 0.671, 0.618, 0.631, 0.734, ]

No. 76
-1  ->   0  [0.851, 0.681, 0.616, 0.630, 0.687, ]

No. 77
-1  ->   0  [0.836, 0.667, 0.619, 0.634, 0.733, ]

No. 78
-1  ->   0  [0.849, 0.673, 0.617, 0.631, 0.734, ]

No. 111
-1  ->   5  [0.376, 0.841, 0.858, 0.839, 0.839, ]

No. 136
-1  ->   0  [0.750, 0.795, 0.688, 0.714, 0.587, ]

No. 143
-1  ->   1  [0.632, 0.921, 0.754, 0.746, 0.775, ]

No. 146
-1  ->   0  [0.809, 0.879, 0.658, 0.653, 0.678, ]

No. 158
 0  ->  -1  [0.766, 0.841, 0.688, 0.571, 0.682, ]

No. 185
-1  ->   0  [0.775, 0.886, 0.701, 0.621, 0.640, ]

No. 195
 0  ->  -1  [0.694, 0.700, 0.573, 0.579, 0.684, ]

No. 197
 0  ->  -1  [0.716, 0.777, 0.694, 0.604, 0.665, ]

No. 219
-1  ->   0  [0.791, 0.680, 0.681, 0.667, 0.673, ]

No. 225
-1  ->   0  [0.720, 0.772, 0.620, 0.691, 0.584, ]

No. 226
-1  ->   1  [0.594, 0.816, 0.750, 0.733, 0.699, ]

No. 265
-1  ->   0  [0.790, 0.857, 0.647, 0.655, 0.671, ]

50番目輪郭は、点数文字でない文字(交換期限の"5月"の文字)で、今年(2022年)のシール台紙でも同じような文字があったので、学習データに追加し、再度学習を実施しておきます。

train_data += [svm_inputs[50]]
train_labels += [svm_labels[50]]

svm = prepare_svm(train_data, train_labels)

result = svm.predict(np.array(svm_inputs, 'float32'))
print_stat(result, svm_labels)

label -1 : {-1: 44, 0: 17, 1:  2, 2:  0, 3:  0, 5:  2, }
label  0 : {-1:  3, 0: 33, 1:  0, 2:  0, 3:  0, 5:  0, }
label  1 : {-1:  0, 0:  0, 1: 88, 2:  0, 3:  0, 5:  0, }
label  2 : {-1:  0, 0:  0, 1:  0, 2: 57, 3:  0, 5:  0, }
label  3 : {-1:  0, 0:  0, 1:  0, 2:  0, 3:  2, 5:  0, }
label  5 : {-1:  0, 0:  0, 1:  0, 2:  0, 3:  0, 5: 42, }

subimgs = copy.deepcopy(subimgs_all)
subctrs = copy.deepcopy(subctrs_all)

for i,(sims,lab,res,img,ctr) in enumerate(zip(svm_inputs, svm_labels, result[1], subimgs, subctrs)):
    if lab != res[0]:
        print('No.', i)
        print('{: }'.format(lab), ' -> ', '{: d}'.format(int(res[0])), ' [',end='')
        for s in sims: print('{:.3f}, '.format(s), end='');
        print(']')
        img2 = cv2.copyMakeBorder(img.copy(), 2,2,2,2, cv2.BORDER_CONSTANT, 0)
        img2 = cv2.drawContours(img2, [ctr], -1, (0,255,0), 1)
        plt.imshow(cv2.cvtColor(img2, cv2.COLOR_BGR2RGB)),plt.xticks([]),plt.yticks([])
        plt.show()

No. 0
-1  ->   0  [0.828, 0.849, 0.684, 0.649, 0.689, ]

No. 4
-1  ->   0  [0.828, 0.785, 0.686, 0.658, 0.688, ]

No. 5
-1  ->   0  [0.829, 0.793, 0.671, 0.622, 0.665, ]

No. 36
-1  ->   0  [0.831, 0.844, 0.685, 0.659, 0.704, ]

No. 39
-1  ->   0  [0.807, 0.778, 0.689, 0.645, 0.686, ]

No. 50
-1  ->   5  [0.473, 0.681, 0.601, 0.702, 0.801, ]

No. 71
-1  ->   0  [0.850, 0.682, 0.619, 0.638, 0.707, ]

No. 74
-1  ->   0  [0.844, 0.680, 0.624, 0.637, 0.732, ]

No. 75
-1  ->   0  [0.837, 0.671, 0.618, 0.631, 0.734, ]

No. 76
-1  ->   0  [0.851, 0.681, 0.616, 0.630, 0.687, ]

No. 77
-1  ->   0  [0.836, 0.667, 0.619, 0.634, 0.733, ]

No. 78
-1  ->   0  [0.849, 0.673, 0.617, 0.631, 0.734, ]

No. 111
-1  ->   5  [0.376, 0.841, 0.858, 0.839, 0.839, ]

No. 136
-1  ->   0  [0.750, 0.795, 0.688, 0.714, 0.587, ]

No. 143
-1  ->   1  [0.632, 0.921, 0.754, 0.746, 0.775, ]

No. 146
-1  ->   0  [0.809, 0.879, 0.658, 0.653, 0.678, ]

No. 158
 0  ->  -1  [0.766, 0.841, 0.688, 0.571, 0.682, ]

No. 185
-1  ->   0  [0.775, 0.886, 0.701, 0.621, 0.640, ]

No. 195
 0  ->  -1  [0.694, 0.700, 0.573, 0.579, 0.684, ]

No. 197
 0  ->  -1  [0.716, 0.777, 0.694, 0.604, 0.665, ]

No. 219
-1  ->   0  [0.791, 0.680, 0.681, 0.667, 0.673, ]

No. 225
-1  ->   0  [0.720, 0.772, 0.620, 0.691, 0.584, ]

No. 226
-1  ->   1  [0.594, 0.816, 0.750, 0.733, 0.699, ]

No. 265
-1  ->   0  [0.790, 0.857, 0.647, 0.655, 0.671, ]

特に学習データ追加をしても変わりませんでした。もしかしたらすでに含まれていたかも。
調整はこんなところで諦めにします。
ということで、テンプレートデータとSVMデータを保存。

svm.save('harupan_data/harupan_svm_220412.dat')

# ctr_list: List of contours for (0, 1, 2, 3, 5)
# pts_idx_list: List of selected point indices for (1, 2, 3, 5)
def save_templates(filename, ctr_list, pts_idx_list):
    with open(filename, mode='w') as f:
        save_data = []
        save_data += [{'num': 0, 'ctr': ctr_list[0].tolist(), 'pts': [0]}]
        for num, ctr, pts_idx in zip([1,2,3,5], ctr_list[1:5], pts_idx_list):
            save_data += [{'num': num, 'ctr': ctr.tolist(), 'pts': pts_idx}]
        json.dump(save_data, f, indent=2)
    return

save_templates('harupan_data/templates2019_220412.json', subctrs1, subctrs1_selected_pts)
save_templates('harupan_data/templates2020_220412.json', subctrs3, subctrs3_selected_pts)
save_templates('harupan_data/templates2021_220412.json', subctrs5, subctrs5_selected_pts)

変更したスクリプト(harupan.py)も以下に全体を示します。

######################################################
# Importing libraries
######################################################
import cv2
import numpy as np
from matplotlib import pyplot as plt
import math
import copy
import random
import json

######################################################
# Detecting contours
######################################################
def reduce_resolution(img, res_th=800):
    h, w, chs = img.shape
    if h > res_th or w > res_th:
        k = float(res_th)/h if w > h else float(res_th)/w
    else:
        k = 1.0
    rtn_img = cv2.resize(img, None, fx=k, fy=k, interpolation=cv2.INTER_AREA)
    return rtn_img

def harupan_binarize(img, sat_th=100):
    hsv = cv2.cvtColor(img, cv2.COLOR_BGR2HSV)
    # Convert hue value (rotation, mask by saturation)
    hsv[:,:,0] = np.where(hsv[:,:,0] < 50, hsv[:,:,0]+180, hsv[:,:,0])
    hsv[:,:,0] = np.where(hsv[:,:,1] < sat_th, 0, hsv[:,:,0])
    # Thresholding with cv2.inRange()
    binary_img = cv2.inRange(hsv[:,:,0], 135, 190)
    return binary_img

def detect_candidate_contours(image, res_th=800, sat_th=100):
    img = reduce_resolution(image, res_th)
    binimg = harupan_binarize(img, sat_th)
    # Retrieve all points on the contours (cv2.CHAIN_APPROX_NONE)
    contours, hierarchy = cv2.findContours(binimg, cv2.RETR_TREE, cv2.CHAIN_APPROX_NONE)
    indices0 = [i for i,hier in enumerate(hierarchy[0,:,:]) if hier[3] == -1]
    indices1 = [i for i,hier in enumerate(hierarchy[0,:,:]) if hier[3] in indices0]
    contours1 = [contours[i] for i in indices1]
    contours1_filtered = [ctr for ctr in contours1 if cv2.contourArea(ctr) > float(res_th)*float(res_th)/4000]
    return contours1_filtered, img

# image: Entire image containing multiple contours
# contours: Contours contained in "image" (Retrieved by cv2.findContours(), the origin is same as "image")
def refine_contours(image, contours):
    subctrs = []
    subimgs = []
    binimgs = []
    thresholds = []
    n_ctrs = []
    for ctr in contours:
        img, _ = create_contour_area_image(image, ctr)
        # Thresholding using G value in BGR format
        thresh, binimg = cv2.threshold(img[:,:,1], 0, 255, cv2.THRESH_BINARY + cv2.THRESH_OTSU)
        # Add black region around thresholded image, to detect contours correctly
        binimg = cv2.copyMakeBorder(binimg, 2,2,2,2, cv2.BORDER_CONSTANT, 0)
        ctrs2, _ = cv2.findContours(binimg, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_NONE)
        max_len = 0
        for ctr2 in ctrs2:
            if max_len <= ctr2.shape[0]:
                max_ctr = ctr2
                max_len = ctr2.shape[0]
        subctrs += [max_ctr]
        subimgs += [img]
        binimgs += [binimg]
        thresholds += [thresh]
        n_ctrs += [len(ctrs2)]
    debug_info = (binimgs, thresholds, n_ctrs)
    return subctrs, subimgs, debug_info

######################################################
# Auxiliary functions
######################################################
def create_contour_area_image(img, ctr):
    x,y,w,h = cv2.boundingRect(ctr)
    rtn_img = img[y:y+h,x:x+w,:].copy()
    rtn_ctr = ctr.copy()
    origin = np.array([x,y])
    for c in rtn_ctr:
        c[0,:] -= origin
    return rtn_img, rtn_ctr

# ctr: Should be output of create_contour_area_image() (Origin of points is the origin of bounding box)
# img_shape: Optional, tuple of (image_height, image_width), if omitted, calculated from ctr
def create_solid_contour(ctr, img_shape=(int(0),int(0))):
    if img_shape == (int(0),int(0)):
        _,_,w,h = cv2.boundingRect(ctr)
    else:
        h,w = img_shape
    img = np.zeros((h,w), 'uint8')
    img = cv2.drawContours(img, [ctr], -1, 255, -1)
    return img

# ctr: Should be output of create_contour_area_image() (Origin of points is the origin of bounding box)
def create_upright_solid_contour(ctr):
    ctr2 = ctr.copy()
    (cx,cy),(w,h),angle = cv2.minAreaRect(ctr2)
    M = cv2.getRotationMatrix2D((cx,cy), angle, 1)
    for i in range(ctr2.shape[0]):
        ctr2[i,0,:] = ( M @ np.array([ctr2[i,0,0], ctr2[i,0,1], 1]) ).astype('int')
    rect = cv2.boundingRect(ctr2)
    img = np.zeros((rect[3],rect[2]), 'uint8')
    ctr2 -= rect[0:2]
    M[:,2] -= rect[0:2]
    img = cv2.drawContours(img, [ctr2], -1, 255,-1)
    return img, M, ctr2


######################################################
# Dataset classes
######################################################
class contour_dataset:
    def __init__(self, ctr):
        self.ctr = ctr.copy()
        self.rrect = cv2.minAreaRect(ctr)
        self.box = cv2.boxPoints(self.rrect)
        self.solid = create_solid_contour(ctr)
        self.pts = np.array([p for p in ctr[:,0,:]])

class template_dataset:
    def __init__(self, ctr, num, selected_idx=[0]):
        self.ctr = ctr.copy()
        self.num = num
        self.rrect = cv2.minAreaRect(ctr)
        self.box = cv2.boxPoints(self.rrect)
        if num == 0:
            self.solid,_,_ = create_upright_solid_contour(ctr)
        else:
            self.solid = create_solid_contour(ctr)
        self.pts = np.array([ctr[idx,0,:] for idx in selected_idx])


######################################################
# ICP
######################################################
# pts: list of 2D points, or ndarray of shape (n,2)
# query: 2D point to find nearest neighbor
def find_nearest_neighbor(pts, query):
    min_distance_sq = float('inf')
    min_idx = 0
    for i, p in enumerate(pts):
        d = np.dot(query - p, query - p)
        if(d < min_distance_sq):
            min_distance_sq = d
            min_idx = i
    return min_idx, np.sqrt(min_distance_sq)

# src, dst: ndarray, shape is (n,2) (n: number of points)
def estimate_affine_2d(src, dst):
    n = min(src.shape[0], dst.shape[0])
    x = dst[0:n].flatten()
    A = np.zeros((2*n,6))
    for i in range(n):
        A[i*2,0] = src[i,0]
        A[i*2,1] = src[i,1]
        A[i*2,2] = 1
        A[i*2+1,3] = src[i,0]
        A[i*2+1,4] = src[i,1]
        A[i*2+1,5] = 1
    M = np.linalg.inv(A.T @ A) @ A.T @ x
    return M.reshape([2,3])

# Find optimum affine matrix using ICP algorithm
# src_pts: ndarray, shape is (n_s,2) (n_s: number of points)
# dst_pts: ndarray, shape is (n_d,2) (n_d: number of points, n_d should be larger or equal to n_s)
# initial_matrix: ndarray, shape is (2,3)
def icp(src_pts, dst_pts, max_iter=20, initial_matrix=np.array([[1.0, 0.0, 0.0], [0.0, 1.0, 0.0]])):
    default_affine_matrix = np.array([[1.0, 0.0, 0.0], [0.0, 1.0, 0.0]])
    if dst_pts.shape[0] < src_pts.shape[0]:
        print("icp: Insufficient destination points")
        return default_affine_matrix, False
    if initial_matrix.shape != (2,3):
        print("icp: Illegal shape of initial_matrix")
        return default_affine_matrix, False
    M = initial_matrix
    # Store indices of the nearest neighbor point of dst_pts to the converted point of src_pts
    nn_idx = []
    for i in range(max_iter):
        nn_idx_tmp = []
        dst_pts_list = [p for p in dst_pts]
        idx_list = list(range(0,dst_pts.shape[0]))
        for p in src_pts:
            p2 = M @ np.array([p[0], p[1], 1])
            idx, d = find_nearest_neighbor(dst_pts_list, p2)
            nn_idx_tmp += [idx_list[idx]]
            del dst_pts_list[idx]
            del idx_list[idx]
        if nn_idx != [] and nn_idx == nn_idx_tmp:
            break
        dst_pts2 = np.zeros_like(src_pts)
        for j,idx in enumerate(nn_idx_tmp):
            dst_pts2[j,:] = dst_pts[idx,:]
        M = estimate_affine_2d(src_pts, dst_pts2)
        nn_idx = nn_idx_tmp
        if i == max_iter -1:
            return M, False
    return M, True


######################################################
# Calculating similarity and determining the number
######################################################
def binary_image_similarity(img1, img2):
    if img1.shape != img2.shape:
        print('binary_image_similarity: Different image size')
        return 0.0
    xor_img = cv2.bitwise_xor(img1, img2)
    return 1.0 - np.float(np.count_nonzero(xor_img)) / (img1.shape[0]*img2.shape[1])

# src, dst: contour_dataset or template_dataset (holding member variables box, solid)
def get_transform_by_rotated_rectangle(src, dst):
    # Rotated patterns are created when starting index is slided
    dst_box2 = np.vstack([dst.box, dst.box])
    max_similarity = 0.0
    max_converted_img = np.zeros((dst.solid.shape[1], dst.solid.shape[0]), 'uint8')
    for i in range(4):
        M = cv2.getAffineTransform(src.box[0:3], dst_box2[i:i+3])
        converted_img = cv2.warpAffine(src.solid, M, dsize=(dst.solid.shape[1], dst.solid.shape[0]), flags=cv2.INTER_NEAREST)
        similarity = binary_image_similarity(converted_img, dst.solid)
        if similarity > max_similarity:
            M_rtn = M
            max_similarity = similarity
            max_converted_img = converted_img
    return M_rtn, max_similarity, max_converted_img

def get_similarity_with_template(target_data, template_data, sim_th_high=0.95, sim_th_low=0.7):
    _,(w1,h1), _ = target_data.rrect
    _,(w2,h2), _ = template_data.rrect
    r = w1/h1 if w1 < h1 else h1/w1
    r = r * h2/w2 if w2 < h2 else r * w2/h2
    M, sim_init, _ = get_transform_by_rotated_rectangle(template_data, target_data)
    if sim_init > sim_th_high or sim_init < sim_th_low or r > 1.4 or r < 0.7:
        dsize = (template_data.solid.shape[1], template_data.solid.shape[0])
        flags = cv2.INTER_NEAREST|cv2.WARP_INVERSE_MAP
        converted_img = cv2.warpAffine(target_data.solid, M, dsize=dsize, flags=flags)
        return sim_init, converted_img
    M, _ = icp(template_data.pts, target_data.pts, initial_matrix=M)
    Minv = cv2.invertAffineTransform(M)
    converted_ctr = np.zeros_like(target_data.ctr)
    for i in range(target_data.ctr.shape[0]):
        converted_ctr[i,0,:] = (Minv[:,0:2] @ target_data.ctr[i,0,:]) + Minv[:,2]
    converted_img = create_solid_contour(converted_ctr, img_shape=template_data.solid.shape)
    val = binary_image_similarity(converted_img, template_data.solid)
    return val, converted_img

def get_similarity_with_template_zero(target_data, template_data):
    dsize = (template_data.solid.shape[1], template_data.solid.shape[0])
    converted_img = cv2.resize(target_data.solid, dsize=dsize, interpolation=cv2.INTER_NEAREST)
    val = binary_image_similarity(converted_img, template_data.solid)
    return val, converted_img

def get_similarities(target, templates):
    similarities = []
    converted_imgs = []
    for tmpl in templates:
        if tmpl.num == 0:
            sim,converted_img = get_similarity_with_template_zero(target, tmpl)
        else:
            sim,converted_img = get_similarity_with_template(target, tmpl)
        similarities += [sim]
        converted_imgs += [converted_img]
    return similarities, converted_imgs

def calc_harupan(img, templates, svm):
    ctrs, resized_img = detect_candidate_contours(img, sat_th=50)
    print('Number of candidates: ', len(ctrs))
    subctrs, _, _ = refine_contours(resized_img, ctrs)
    subctr_datasets = [contour_dataset(ctr) for ctr in subctrs]
    ########
    #### Simple code
    # similarities = [get_similarities(d, templates)[0] for d in subctr_datasets]
    #### Code printing progress
    similarities = []
    for i,d in enumerate(subctr_datasets):
        print(i, end=' ')
        similarities += [get_similarities(d, templates)[0]]
    print('')
    ########
    _, result = svm.predict(np.array(similarities, 'float32'))
    result = result.astype('int')
    score = 0.0
    texts = {0:'0', 1:'1', 2:'2', 3:'3', 5:'.5'}
    font = cv2.FONT_HERSHEY_SIMPLEX
    for res, ctr in zip(result, ctrs):
        if res[0] == 5:
            score += 0.5
        elif res[0] != -1:
            score += res[0]
        
        # Annotating recognized numbers for confirmation
        if res[0] != -1:
            resized_img = cv2.drawContours(resized_img, [ctr], -1, (0,255,0), 3)
            x,y,_,_ = cv2.boundingRect(ctr)
            resized_img = cv2.putText(resized_img, texts[res[0]], (x,y), font, 1, (230,230,0), 5)
    return score, resized_img

######################################################
# Loading template data and SVM model
######################################################
def load_svm(filename):
    return cv2.ml.SVM_load(filename)

def load_templates(filename):
    with open(filename, mode='r') as f:
        load_data = json.load(f)
        templates_rtn = []
        for d in load_data:
            templates_rtn += [template_dataset(np.array(d['ctr']), d['num'], d['pts'])]
    return templates_rtn

再確認

変更したテンプレートデータ、SVMデータ、スクリプトできちんと動くかどうか、一応いくつかの画像で確認。

途中バグがあり、スクリプトを修正しながらやっていましたが、Jupyter notebookではカーネルの再起動をしないと基本的にはスクリプトの変更は反映されないとのこと。

調べてみると、importlibで対応している例がありました。
jupyter上でスムーズにモジュールを再読み込みする

import importlib
importlib.reload(harupan_data.harupan)
from harupan_data.harupan import *

svm = cv2.ml.SVM_load('harupan_data/harupan_svm_220412.dat')

templates2019 = load_templates('harupan_data/templates2019_220412.json')
templates2020 = load_templates('harupan_data/templates2020_220412.json')
templates2021 = load_templates('harupan_data/templates2021_220412.json')

import time

t0 = time.time()
score, result_img = calc_harupan(img1, templates2019, svm)
t1 = time.time()
print('Score: ', score)
print('Elapsed time: ', t1 - t0)
plt.figure(figsize=(6.4,4.8), dpi=200)
plt.imshow(cv2.cvtColor(result_img, cv2.COLOR_BGR2RGB)), plt.xticks([]), plt.yticks([])
plt.show()

Number of candidates:  36
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 
Score:  26.0
Elapsed time:  9.816221714019775

t0 = time.time()
score, result_img = calc_harupan(img8, templates2021, svm)
t1 = time.time()
print('Score: ', score)
print('Elapsed time: ', t1 - t0)
plt.figure(figsize=(6.4,4.8), dpi=200)
plt.imshow(cv2.cvtColor(result_img, cv2.COLOR_BGR2RGB)), plt.xticks([]), plt.yticks([])
plt.show()

Number of candidates:  25
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 
Score:  26.0
Elapsed time:  3.784372091293335

t0 = time.time()
score, result_img = calc_harupan(img9, templates2021, svm)
t1 = time.time()
print('Score: ', score)
print('Elapsed time: ', t1 - t0)
plt.figure(figsize=(6.4,4.8), dpi=200)
plt.imshow(cv2.cvtColor(result_img, cv2.COLOR_BGR2RGB)), plt.xticks([]), plt.yticks([])
plt.show()

Number of candidates:  24
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 
Score:  26.0
Elapsed time:  6.287338733673096

うまくいっていなかったPCカメラ画像でも、なんとか点数が正しく認識、計算できるようになりました。

今回はここまで

今回やったことをまとめると、

• 最初の2値化時の彩度閾値を変えることで、点数文字輪郭を正しく取得できるようになった
• 輪郭検出後、検出した輪郭周辺の小画像で再度2値化、輪郭検出を行うことで、輪郭形状を改善できるようになった
• 変更した処理でテンプレートデータ、SVMデータの再生成を行った

というところ。

次回、もう1点だけ変更したいところがあるので、それをやってみます。

OpenCVやってみる - 40. カメラ画像取得

春のパン祭り点数集計も仕上げの段階ですが、今回はPCのカメラ画像を取得できるようにしたいと思います。

カメラから画像取得

OpenCVチュートリアルの最初のほうでやっています。
これを試してみます。

https://docs.opencv.org/4.x/dd/d43/tutorial_py_video_display.html

やることは、cv2.VideoCapture()でVideoCaptureオブジェクトを作り、read()で画像データを1枚得る、というところです。

import cv2
%matplotlib inline
from matplotlib import pyplot as plt

cap = cv2.VideoCapture(0)
if not cap.isOpened():
    print('Cannot open camera')
else:
    print('Camera opened')

Camera opened

ret, frame = cap.read()
if not ret:
    print("Can't receive frame")
else:
    plt.imshow(cv2.cvtColor(frame, cv2.COLOR_BGR2RGB))
    plt.show()

print(frame.shape)

(480, 640, 3)

カメラ画像はあっさり取得できました。 ここでは、今年の春のパン祭りシール台紙を撮影してみました。

解像度はVGAサイズのようです。
思ったより小さい…大丈夫か？

この画像で点数計算をやってみます。
文字のフォントとしては、2021年のものと同じように見えるので、2021年のテンプレートデータを使います。

from harupan_data.harupan import *

svm = load_svm('harupan_data/harupan_svm.dat')
templates2021 = load_templates('harupan_data/templates2021.json')

import time

t0 = time.time()
score, result_img = calc_harupan(frame, templates2021, svm)
t1 = time.time()
print('Score: ', score)
print('Elapsed time: ', t1 - t0)
plt.figure(figsize=(6.4,4.8), dpi=200)
plt.imshow(cv2.cvtColor(result_img, cv2.COLOR_BGR2RGB)), plt.xticks([]), plt.yticks([])
plt.show()

    Number of candidates:  18
    0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 
    Score:  19.5
    Elapsed time:  5.427248001098633

ちょっとうまくいっていない…

撮影し直してみる。

ret, frame2 = cap.read()
if not ret:
    print("Can't receive frame")
else:
    plt.imshow(cv2.cvtColor(frame2, cv2.COLOR_BGR2RGB))
    plt.show()

t0 = time.time()
score, result_img = calc_harupan(frame2, templates2021, svm)
t1 = time.time()
print('Score: ', score)
print('Elapsed time: ', t1 - t0)
plt.figure(figsize=(6.4,4.8), dpi=200)
plt.imshow(cv2.cvtColor(result_img, cv2.COLOR_BGR2RGB)), plt.xticks([]), plt.yticks([])
plt.show()

    Number of candidates:  33
    0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 
    Score:  24.5
    Elapsed time:  6.267446279525757

結構アップにしてみましたが、いまいち。

• "2.5"の検出がうまくいっていない
  "2"、"."、"5"の間の間隔が小さくて、輪郭検出に失敗していると思われます。
• "0.5"の検出もうまくいっていない
  点数文字でない"5"の輪郭データを学習に入れたので、厳しめに見られているのか。
• シール下部の"ヤマザキ"の文字が"1"に認識されてしまっている

一応画像データは保存しておこう。

cv2.imwrite('harupan_220330_1.jpg', frame)
cv2.imwrite('harupan_220330_2.jpg', frame2)

True

ここまで

短いですが、今回ここまでにしておきます。

OpenCVやってみる - 39. 点数計算処理実装

今回は、点数計算の実アプリケーション作成に向けて、以下の処理を検討します。

• 画像1枚を受け取ってからの一連の処理
• 点数計算

処理フロー

点数計算アプリケーションは、ストリーミングされている画像データから、1枚取り出して計算処理を実施・表示、というのを繰り返すイメージで考えます。
処理の流れは、

• 画像を1枚取得
• 点数文字の候補となる輪郭を検出
• 各輪郭について、一致度ベクトルを計算
• SVMに通して、どの数字なのか、それとも数字の輪郭でないのか判定
• 判定結果の数字を足して、総点数を計算
• 結果の表示

最後の結果の表示ですが、おそらく誤認識は発生するので、総点数を表示するだけではなく、どこの輪郭を何点と認識したか、ということも表示したいと思います。

下準備

今までは、作成した処理関数の定義をJupyter notebookが変わるたびにやっていましたが、前回スクリプト(harupan.py)にまとめたので、必要な準備はだいぶ減ります。

スクリプトの実行は、importを使えばいいとのこと。

https://note.nkmk.me/python-import-usage/

スクリプトのディレクトリと実行ディレクトリが分かれている場合の対応:

https://techacademy.jp/magazine/23279

実行するのは、

• スクリプト読み込み
• テンプレートデータ、SVMデータの読み込み
• 処理対象画像の読み込み

といったところです。

from harupan_data.harupan import *

svm = load_svm('harupan_data/harupan_svm.dat')
templates2019 = load_templates('harupan_data/templates2019.json')
templates2020 = load_templates('harupan_data/templates2020.json')
templates2021 = load_templates('harupan_data/templates2021.json')

img1 = cv2.imread('harupan_190428_1.jpg')
img2 = cv2.imread('harupan_190428_2.jpg')
img3 = cv2.imread('harupan_200317_1.jpg')
img4 = cv2.imread('harupan_210227_2.jpg')
img5 = cv2.imread('harupan_210402_1.jpg')
img6 = cv2.imread('harupan_210402_2.jpg')
img7 = cv2.imread('harupan_210414_1.jpg')

一連の計算処理実装

実際に処理を作っていきます。

この中で点数計算も行っていきます。

• "1"、"2"、"3"の数字を検出したら、その数をそのまま点数として足す
• "5"を検出したら、0.5点を足す

def calc_harupan(img, templates, svm):
    ctrs, resized_img = detect_candidate_contours(img)
    print('Number of candidates: ', len(ctrs))
    subctr_datasets = [contour_dataset(create_contour_area_image(resized_img, ctr)[1]) for ctr in ctrs]
    ########
    #### Simple code
    # similarities = [get_similarities(d, templates)[0] for d in subctr_datasets]
    #### Code printing progress
    similarities = []
    for i,d in enumerate(subctr_datasets):
        print(i, end=' ')
        similarities += [get_similarities(d, templates)[0]]
    print('')
    ########
    _, result = svm.predict(np.array(similarities, 'float32'))
    result = result.astype('int')
    score = 0.0
    texts = {0:'0', 1:'1', 2:'2', 3:'3', 5:'.5'}
    font = cv2.FONT_HERSHEY_SIMPLEX
    for res, ctr in zip(result, ctrs):
        if res[0] == 5:
            score += 0.5
        elif res[0] != -1:
            score += res[0]
        
        # Annotating recognized numbers for confirmation
        if res[0] != -1:
            resized_img = cv2.drawContours(resized_img, [ctr], -1, (0,255,0), 3)
            x,y,_,_ = cv2.boundingRect(ctr)
            resized_img = cv2.putText(resized_img, texts[res[0]], (x,y), font, 1, (230,230,0), 5)
    return score, resized_img

処理実施

処理時間計測も合わせて実施します。

import time

t0 = time.time()
score, result_img = calc_harupan(img1, templates2019, svm)
t1 = time.time()
print('Score: ', score)
print('Elapsed time: ', t1 - t0)
plt.figure(figsize=(6.4,4.8), dpi=200)
plt.imshow(cv2.cvtColor(result_img, cv2.COLOR_BGR2RGB)), plt.xticks([]), plt.yticks([])
plt.show()

    Number of candidates:  38
    0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 
    Score:  26.0
    Elapsed time:  10.60127878189087

まあまあな感じです。 この画像では、シールのてかりで点数シール(0.5点)が1つ認識に失敗しています。また、点数ではない"5"の文字を計算に入れてしまっています。

たまたま計算結果は合ってしまったが…

処理時間は10秒近くかかっているので、あまり実用に堪える感じではないかな…

他の画像もやってみます。

t0 = time.time()
score, result_img = calc_harupan(img2, templates2019, svm)
t1 = time.time()
print('Score: ', score)
print('Elapsed time: ', t1 - t0)
plt.figure(figsize=(6.4,4.8), dpi=200)
plt.imshow(cv2.cvtColor(result_img, cv2.COLOR_BGR2RGB)), plt.xticks([]), plt.yticks([])
plt.show()

    Number of candidates:  46
    0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 
    Score:  26.0
    Elapsed time:  11.16847562789917

t0 = time.time()
score, result_img = calc_harupan(img3, templates2020, svm)
t1 = time.time()
print('Score: ', score)
print('Elapsed time: ', t1 - t0)
plt.figure(figsize=(6.4,4.8), dpi=200)
plt.imshow(cv2.cvtColor(result_img, cv2.COLOR_BGR2RGB)), plt.xticks([]), plt.yticks([])
plt.show()

    Number of candidates:  48
    0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 
    Score:  25.0
    Elapsed time:  7.951741933822632

t0 = time.time()
score, result_img = calc_harupan(img4, templates2021, svm)
t1 = time.time()
print('Score: ', score)
print('Elapsed time: ', t1 - t0)
plt.figure(figsize=(6.4,4.8), dpi=200)
plt.imshow(cv2.cvtColor(result_img, cv2.COLOR_BGR2RGB)), plt.xticks([]), plt.yticks([])
plt.show()

    Number of candidates:  41
    0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 
    Score:  20.0
    Elapsed time:  6.7907874584198

t0 = time.time()
score, result_img = calc_harupan(img5, templates2021, svm)
t1 = time.time()
print('Score: ', score)
print('Elapsed time: ', t1 - t0)
plt.figure(figsize=(6.4,4.8), dpi=200)
plt.imshow(cv2.cvtColor(result_img, cv2.COLOR_BGR2RGB)), plt.xticks([]), plt.yticks([])
plt.show()

    Number of candidates:  35
    0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 
    Score:  28.0
    Elapsed time:  6.287672996520996

t0 = time.time()
score, result_img = calc_harupan(img6, templates2021, svm)
t1 = time.time()
print('Score: ', score)
print('Elapsed time: ', t1 - t0)
plt.figure(figsize=(6.4,4.8), dpi=200)
plt.imshow(cv2.cvtColor(result_img, cv2.COLOR_BGR2RGB)), plt.xticks([]), plt.yticks([])
plt.show()

    Number of candidates:  33
    0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 
    Score:  28.0
    Elapsed time:  4.226221799850464

t0 = time.time()
score, result_img = calc_harupan(img7, templates2021, svm)
t1 = time.time()
print('Score: ', score)
print('Elapsed time: ', t1 - t0)
plt.figure(figsize=(6.4,4.8), dpi=200)
plt.imshow(cv2.cvtColor(result_img, cv2.COLOR_BGR2RGB)), plt.xticks([]), plt.yticks([])
plt.show()

    Number of candidates:  24
    0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 
    Score:  24.0
    Elapsed time:  6.231278896331787

1つ目の画像以外は、点数計算は合っていそうです。
処理時間はそれほど変わらず、やっぱりかなりかかっています。

スクリプト更新

点数計算はこれでいいとして、この点数計算処理をスクリプト(harupan.py)に追加しておきます。
以下に全部再掲します。

######################################################
# Importing libraries
######################################################
import cv2
import numpy as np
from matplotlib import pyplot as plt
import math
import copy
import random
import json

######################################################
# Detecting contours
######################################################
def detect_candidate_contours(image, res_th=800):
    h, w, chs = image.shape
    if h > res_th or w > res_th:
        k = float(res_th)/h if w > h else float(res_th)/w
    else:
        k = 1.0
    img = cv2.resize(image, None, fx=k, fy=k, interpolation=cv2.INTER_AREA)
    hsv = cv2.cvtColor(img, cv2.COLOR_BGR2HSV)
    # Convert hue value (rotation, mask by saturation)
    hsv[:,:,0] = np.where(hsv[:,:,0] < 50, hsv[:,:,0]+180, hsv[:,:,0])
    hsv[:,:,0] = np.where(hsv[:,:,1] < 100, 0, hsv[:,:,0])
    # Thresholding with cv2.inRange()
    th_hue = cv2.inRange(hsv[:,:,0], 135, 190)
    # Retrieve all points on the contours (cv2.CHAIN_APPROX_NONE)
    contours, hierarchy = cv2.findContours(th_hue, cv2.RETR_TREE, cv2.CHAIN_APPROX_NONE)
    indices0 = [i for i,hier in enumerate(hierarchy[0,:,:]) if hier[3] == -1]
    indices1 = [i for i,hier in enumerate(hierarchy[0,:,:]) if hier[3] in indices0]
    contours1 = [contours[i] for i in indices1]
    contours1_filtered = [ctr for ctr in contours1 if cv2.contourArea(ctr) > float(res_th)*float(res_th)/4000]
    return contours1_filtered, img


######################################################
# Auxiliary functions
######################################################
def create_contour_area_image(img, ctr):
    x,y,w,h = cv2.boundingRect(ctr)
    rtn_img = img[y:y+h,x:x+w,:].copy()
    rtn_ctr = ctr.copy()
    origin = np.array([x,y])
    for c in rtn_ctr:
        c[0,:] -= origin
    return rtn_img, rtn_ctr

# ctr: Should be output of create_contour_area_image() (Origin of points is the origin of bounding box)
# img_shape: Optional, tuple of (image_height, image_width), if omitted, calculated from ctr
def create_solid_contour(ctr, img_shape=(int(0),int(0))):
    if img_shape == (int(0),int(0)):
        _,_,w,h = cv2.boundingRect(ctr)
    else:
        h,w = img_shape
    img = np.zeros((h,w), 'uint8')
    img = cv2.drawContours(img, [ctr], -1, 255, -1)
    return img

# ctr: Should be output of create_contour_area_image() (Origin of points is the origin of bounding box)
def create_upright_solid_contour(ctr):
    (cx,cy),(w,h),angle = cv2.minAreaRect(ctr)
    M = cv2.getRotationMatrix2D((cx,cy), angle, 1)
    for i in range(ctr.shape[0]):
        ctr[i,0,:] = ( M @ np.array([ctr[i,0,0], ctr[i,0,1], 1]) ).astype('int')
    rect = cv2.boundingRect(ctr)
    img = np.zeros((rect[3],rect[2]), 'uint8')
    ctr -= rect[0:2]
    M[:,2] -= rect[0:2]
    img = cv2.drawContours(img, [ctr], -1, 255,-1)
    return img, M, ctr


######################################################
# Dataset classes
######################################################
class contour_dataset:
    def __init__(self, ctr):
        self.ctr = ctr.copy()
        self.rrect = cv2.minAreaRect(ctr)
        self.box = cv2.boxPoints(self.rrect)
        self.solid = create_solid_contour(ctr)
        self.pts = np.array([p for p in ctr[:,0,:]])

class template_dataset:
    def __init__(self, ctr, num, selected_idx=[0]):
        self.ctr = ctr.copy()
        self.num = num
        self.rrect = cv2.minAreaRect(ctr)
        self.box = cv2.boxPoints(self.rrect)
        if num == 0:
            self.solid,_,_ = create_upright_solid_contour(ctr)
        else:
            self.solid = create_solid_contour(ctr)
        self.pts = np.array([ctr[idx,0,:] for idx in selected_idx])


######################################################
# ICP
######################################################
# pts: list of 2D points, or ndarray of shape (n,2)
# query: 2D point to find nearest neighbor
def find_nearest_neighbor(pts, query):
    min_distance_sq = float('inf')
    min_idx = 0
    for i, p in enumerate(pts):
        d = np.dot(query - p, query - p)
        if(d < min_distance_sq):
            min_distance_sq = d
            min_idx = i
    return min_idx, np.sqrt(min_distance_sq)

# src, dst: ndarray, shape is (n,2) (n: number of points)
def estimate_affine_2d(src, dst):
    n = min(src.shape[0], dst.shape[0])
    x = dst[0:n].flatten()
    A = np.zeros((2*n,6))
    for i in range(n):
        A[i*2,0] = src[i,0]
        A[i*2,1] = src[i,1]
        A[i*2,2] = 1
        A[i*2+1,3] = src[i,0]
        A[i*2+1,4] = src[i,1]
        A[i*2+1,5] = 1
    M = np.linalg.inv(A.T @ A) @ A.T @ x
    return M.reshape([2,3])

# Find optimum affine matrix using ICP algorithm
# src_pts: ndarray, shape is (n_s,2) (n_s: number of points)
# dst_pts: ndarray, shape is (n_d,2) (n_d: number of points, n_d should be larger or equal to n_s)
# initial_matrix: ndarray, shape is (2,3)
def icp(src_pts, dst_pts, max_iter=20, initial_matrix=np.array([[1.0, 0.0, 0.0], [0.0, 1.0, 0.0]])):
    default_affine_matrix = np.array([[1.0, 0.0, 0.0], [0.0, 1.0, 0.0]])
    if dst_pts.shape[0] < src_pts.shape[0]:
        print("icp: Insufficient destination points")
        return default_affine_matrix, False
    if initial_matrix.shape != (2,3):
        print("icp: Illegal shape of initial_matrix")
        return default_affine_matrix, False
    M = initial_matrix
    # Store indices of the nearest neighbor point of dst_pts to the converted point of src_pts
    nn_idx = []
    for i in range(max_iter):
        nn_idx_tmp = []
        dst_pts_list = [p for p in dst_pts]
        idx_list = list(range(0,dst_pts.shape[0]))
        for p in src_pts:
            p2 = M @ np.array([p[0], p[1], 1])
            idx, d = find_nearest_neighbor(dst_pts_list, p2)
            nn_idx_tmp += [idx_list[idx]]
            del dst_pts_list[idx]
            del idx_list[idx]
        if nn_idx != [] and nn_idx == nn_idx_tmp:
            break
        dst_pts2 = np.zeros_like(src_pts)
        for j,idx in enumerate(nn_idx_tmp):
            dst_pts2[j,:] = dst_pts[idx,:]
        M = estimate_affine_2d(src_pts, dst_pts2)
        nn_idx = nn_idx_tmp
        if i == max_iter -1:
            return M, False
    return M, True


######################################################
# Calculating similarity and determining the number
######################################################
def binary_image_similarity(img1, img2):
    if img1.shape != img2.shape:
        print('binary_image_similarity: Different image size')
        return 0.0
    xor_img = cv2.bitwise_xor(img1, img2)
    return 1.0 - np.float(np.count_nonzero(xor_img)) / (img1.shape[0]*img2.shape[1])

# src, dst: contour_dataset or template_dataset (holding member variables box, solid)
def get_transform_by_rotated_rectangle(src, dst):
    # Rotated patterns are created when starting index is slided
    dst_box2 = np.vstack([dst.box, dst.box])
    max_similarity = 0.0
    max_converted_img = np.zeros((dst.solid.shape[1], dst.solid.shape[0]), 'uint8')
    for i in range(4):
        M = cv2.getAffineTransform(src.box[0:3], dst_box2[i:i+3])
        converted_img = cv2.warpAffine(src.solid, M, dsize=(dst.solid.shape[1], dst.solid.shape[0]), flags=cv2.INTER_NEAREST)
        similarity = binary_image_similarity(converted_img, dst.solid)
        if similarity > max_similarity:
            M_rtn = M
            max_similarity = similarity
            max_converted_img = converted_img
    return M_rtn, max_similarity, max_converted_img

def get_similarity_with_template(target_data, template_data, sim_th_high=0.95, sim_th_low=0.7):
    _,(w1,h1), _ = target_data.rrect
    _,(w2,h2), _ = template_data.rrect
    r = w1/h1 if w1 < h1 else h1/w1
    r = r * h2/w2 if w2 < h2 else r * w2/h2
    M, sim_init, _ = get_transform_by_rotated_rectangle(template_data, target_data)
    if sim_init > sim_th_high or sim_init < sim_th_low or r > 1.4 or r < 0.7:
        dsize = (template_data.solid.shape[1], template_data.solid.shape[0])
        flags = cv2.INTER_NEAREST|cv2.WARP_INVERSE_MAP
        converted_img = cv2.warpAffine(target_data.solid, M, dsize=dsize, flags=flags)
        return sim_init, converted_img
    M, _ = icp(template_data.pts, target_data.pts, initial_matrix=M)
    Minv = cv2.invertAffineTransform(M)
    converted_ctr = np.zeros_like(target_data.ctr)
    for i in range(target_data.ctr.shape[0]):
        converted_ctr[i,0,:] = (Minv[:,0:2] @ target_data.ctr[i,0,:]) + Minv[:,2]
    converted_img = create_solid_contour(converted_ctr, img_shape=template_data.solid.shape)
    val = binary_image_similarity(converted_img, template_data.solid)
    return val, converted_img

def get_similarity_with_template_zero(target_data, template_data):
    dsize = (template_data.solid.shape[1], template_data.solid.shape[0])
    converted_img = cv2.resize(target_data.solid, dsize=dsize, interpolation=cv2.INTER_NEAREST)
    val = binary_image_similarity(converted_img, template_data.solid)
    return val, converted_img

def get_similarities(target, templates):
    similarities = []
    converted_imgs = []
    for tmpl in templates:
        if tmpl.num == 0:
            sim,converted_img = get_similarity_with_template_zero(target, tmpl)
        else:
            sim,converted_img = get_similarity_with_template(target, tmpl)
        similarities += [sim]
        converted_imgs += [converted_img]
    return similarities, converted_imgs

def calc_harupan(img, templates, svm):
    ctrs, resized_img = detect_candidate_contours(img)
    print('Number of candidates: ', len(ctrs))
    subctr_datasets = [contour_dataset(create_contour_area_image(resized_img, ctr)[1]) for ctr in ctrs]
    ########
    #### Simple code
    # similarities = [get_similarities(d, templates)[0] for d in subctr_datasets]
    #### Code printing progress
    similarities = []
    for i,d in enumerate(subctr_datasets):
        print(i, end=' ')
        similarities += [get_similarities(d, templates)[0]]
    print('')
    ########
    _, result = svm.predict(np.array(similarities, 'float32'))
    result = result.astype('int')
    score = 0.0
    texts = {0:'0', 1:'1', 2:'2', 3:'3', 5:'.5'}
    font = cv2.FONT_HERSHEY_SIMPLEX
    for res, ctr in zip(result, ctrs):
        if res[0] == 5:
            score += 0.5
        elif res[0] != -1:
            score += res[0]
        
        # Annotating recognized numbers for confirmation
        if res[0] != -1:
            resized_img = cv2.drawContours(resized_img, [ctr], -1, (0,255,0), 3)
            x,y,_,_ = cv2.boundingRect(ctr)
            resized_img = cv2.putText(resized_img, texts[res[0]], (x,y), font, 1, (230,230,0), 5)
    return score, resized_img

######################################################
# Loading template data and SVM model
######################################################
def load_svm(filename):
    return cv2.ml.SVM_load(filename)

def load_templates(filename):
    with open(filename, mode='r') as f:
        load_data = json.load(f)
        templates_rtn = []
        for d in load_data:
            templates_rtn += [template_dataset(np.array(d['ctr']), d['num'], d['pts'])]
    return templates_rtn

以上

なんとか形になりそうです。

今考えているのは、PCについているWebカメラで春のパン祭りシール台紙を撮影、画像取得して点数計算する、ということです。
今年の春のパン祭りのシールも集めていて、既に台紙2枚目に入っているので、データとしては準備OK！
わくわく。

OpenCVやってみる-38. 処理の調整2

春のパン祭り点数文字認識処理を調整していますが、なかなか完璧にはいかず…

調整は今回までにして、実際のアプリケーションを作っていく方向に進めたいと思います。

今回変更したこと

色々調整していて、以下変更しました。

• 数字テンプレートとする輪郭データを、cv2.minAreaRect()での角度を元にまっすぐに直しておく。(効果はあまり出なかったが、処理を整理できたので、この変更版処理を使っておきたい)
• ICP処理の中で、最近傍点探索処理があったが、この高速化を行った。
• 点数文字認識処理の区切り方を修正した。
• 一致度計算では、cv2.matchTemplate()関数を使っていたが、2つのサイズの一致した2値画像を比較したい、というだけなので、それに合わせた処理に変更。パフォーマンスは確認していないが、速くなっているんではないかと。

今まで作成した処理

色々と変更しながらやってきたので、現状のバージョンを以下並べていきます。

ライブラリインポート、画像データ読み込み

import cv2
import numpy as np
%matplotlib inline
from matplotlib import pyplot as plt
import math
import copy
import random

img1 = cv2.imread('harupan_190428_1.jpg')
img2 = cv2.imread('harupan_190428_2.jpg')
img3 = cv2.imread('harupan_200317_1.jpg')
img4 = cv2.imread('harupan_210227_2.jpg')
img5 = cv2.imread('harupan_210402_1.jpg')
img6 = cv2.imread('harupan_210402_2.jpg')
img7 = cv2.imread('harupan_210414_1.jpg')

点数文字候補輪郭の検出

def detect_candidate_contours(image, res_th=800):
    h, w, chs = image.shape
    if h > res_th or w > res_th:
        k = float(res_th)/h if w > h else float(res_th)/w
    else:
        k = 1.0
    img = cv2.resize(image, None, fx=k, fy=k, interpolation=cv2.INTER_AREA)
    hsv = cv2.cvtColor(img, cv2.COLOR_BGR2HSV)
    # Convert hue value (rotation, mask by saturation)
    hsv[:,:,0] = np.where(hsv[:,:,0] < 50, hsv[:,:,0]+180, hsv[:,:,0])
    hsv[:,:,0] = np.where(hsv[:,:,1] < 100, 0, hsv[:,:,0])
    # Thresholding with cv2.inRange()
    th_hue = cv2.inRange(hsv[:,:,0], 135, 190)
    # Retrieve all points on the contours (cv2.CHAIN_APPROX_NONE)
    contours, hierarchy = cv2.findContours(th_hue, cv2.RETR_TREE, cv2.CHAIN_APPROX_NONE)
    indices0 = [i for i,hier in enumerate(hierarchy[0,:,:]) if hier[3] == -1]
    indices1 = [i for i,hier in enumerate(hierarchy[0,:,:]) if hier[3] in indices0]
    contours1 = [contours[i] for i in indices1]
    contours1_filtered = [ctr for ctr in contours1 if cv2.contourArea(ctr) > float(res_th)*float(res_th)/4000]
    return contours1_filtered, img

補助処理

• 輪郭周辺の小画像作成
  輪郭周辺の画像領域の切り出し、画像と輪郭自体の原点もその領域の原点に直す。
  比較対象データの作成が一つの目的、もう一つの目的はデバッグ。
• 輪郭の塗りつぶし画像作成
  テンプレートデータ作成用、比較用データ作成用

def create_contour_area_image(img, ctr):
    x,y,w,h = cv2.boundingRect(ctr)
    rtn_img = img[y:y+h,x:x+w,:].copy()
    rtn_ctr = ctr.copy()
    origin = np.array([x,y])
    for c in rtn_ctr:
        c[0,:] -= origin
    return rtn_img, rtn_ctr

# ctr: Should be output of create_contour_area_image() (Origin of points is the origin of bounding box)
# img_shape: Optional, tuple of (image_height, image_width), if omitted, calculated from ctr
def create_solid_contour(ctr, img_shape=(int(0),int(0))):
    if img_shape == (int(0),int(0)):
        _,_,w,h = cv2.boundingRect(ctr)
    else:
        h,w = img_shape
    img = np.zeros((h,w), 'uint8')
    img = cv2.drawContours(img, [ctr], -1, 255, -1)
    return img

# ctr: Should be output of create_contour_area_image() (Origin of points is the origin of bounding box)
def create_upright_solid_contour(ctr):
    (cx,cy),(w,h),angle = cv2.minAreaRect(ctr)
    M = cv2.getRotationMatrix2D((cx,cy), angle, 1)
    for i in range(ctr.shape[0]):
        ctr[i,0,:] = ( M @ np.array([ctr[i,0,0], ctr[i,0,1], 1]) ).astype('int')
    rect = cv2.boundingRect(ctr)
    img = np.zeros((rect[3],rect[2]), 'uint8')
    ctr -= rect[0:2]
    M[:,2] -= rect[0:2]
    img = cv2.drawContours(img, [ctr], -1, 255,-1)
    return img, M, ctr

点数文字認識処理の整理

データセット

各輪郭について、処理の中で何度か使われる生成データ。
クラスでまとめておく。

class contour_dataset:
    def __init__(self, ctr):
        self.ctr = ctr.copy()
        self.rrect = cv2.minAreaRect(ctr)
        self.box = cv2.boxPoints(self.rrect)
        self.solid = create_solid_contour(ctr)
        self.pts = np.array([p for p in ctr[:,0,:]])

class template_dataset:
    def __init__(self, ctr, num, selected_idx=[0]):
        self.ctr = ctr.copy()
        self.num = num
        self.rrect = cv2.minAreaRect(ctr)
        self.box = cv2.boxPoints(self.rrect)
        if num == 0:
            self.solid,_,_ = create_upright_solid_contour(ctr)
        else:
            self.solid = create_solid_contour(ctr)
        self.pts = np.array([ctr[idx,0,:] for idx in selected_idx])

ICP処理

補助処理と本体。

• 最近傍点探索処理は高速化版。
  2点間の距離を計算していましたが、そうするとルート計算が入って計算が重くなるかと。
  距離の2乗の計算であれば計算が軽く、また、大小比較をするだけなら2乗したものでも問題ないので、そのように変更しました。

# pts: list of 2D points, or ndarray of shape (n,2)
# query: 2D point to find nearest neighbor
def find_nearest_neighbor(pts, query):
    min_distance_sq = float('inf')
    min_idx = 0
    for i, p in enumerate(pts):
        d = np.dot(query - p, query - p)
        if(d < min_distance_sq):
            min_distance_sq = d
            min_idx = i
    return min_idx, np.sqrt(min_distance_sq)

# src, dst: ndarray, shape is (n,2) (n: number of points)
def estimate_affine_2d(src, dst):
    n = min(src.shape[0], dst.shape[0])
    x = dst[0:n].flatten()
    A = np.zeros((2*n,6))
    for i in range(n):
        A[i*2,0] = src[i,0]
        A[i*2,1] = src[i,1]
        A[i*2,2] = 1
        A[i*2+1,3] = src[i,0]
        A[i*2+1,4] = src[i,1]
        A[i*2+1,5] = 1
    M = np.linalg.inv(A.T @ A) @ A.T @ x
    return M.reshape([2,3])

# Find optimum affine matrix using ICP algorithm
# src_pts: ndarray, shape is (n_s,2) (n_s: number of points)
# dst_pts: ndarray, shape is (n_d,2) (n_d: number of points, n_d should be larger or equal to n_s)
# initial_matrix: ndarray, shape is (2,3)
def icp(src_pts, dst_pts, max_iter=20, initial_matrix=np.array([[1.0, 0.0, 0.0], [0.0, 1.0, 0.0]])):
    default_affine_matrix = np.array([[1.0, 0.0, 0.0], [0.0, 1.0, 0.0]])
    if dst_pts.shape[0] < src_pts.shape[0]:
        print("icp: Insufficient destination points")
        return default_affine_matrix, False
    if initial_matrix.shape != (2,3):
        print("icp: Illegal shape of initial_matrix")
        return default_affine_matrix, False
    M = initial_matrix
    # Store indices of the nearest neighbor point of dst_pts to the converted point of src_pts
    nn_idx = []
    for i in range(max_iter):
        nn_idx_tmp = []
        dst_pts_list = [p for p in dst_pts]
        idx_list = list(range(0,dst_pts.shape[0]))
        for p in src_pts:
            p2 = M @ np.array([p[0], p[1], 1])
            idx, d = find_nearest_neighbor(dst_pts_list, p2)
            nn_idx_tmp += [idx_list[idx]]
            del dst_pts_list[idx]
            del idx_list[idx]
        if nn_idx != [] and nn_idx == nn_idx_tmp:
            break
        dst_pts2 = np.zeros_like(src_pts)
        for j,idx in enumerate(nn_idx_tmp):
            dst_pts2[j,:] = dst_pts[idx,:]
        M = estimate_affine_2d(src_pts, dst_pts2)
        nn_idx = nn_idx_tmp
        if i == max_iter -1:
            return M, False
    return M, True

一致度計算処理

今まで使っていたものから少しバグ修正(デバッグ用に返す画像が間違っていた)があるぐらい。

def binary_image_similarity(img1, img2):
    if img1.shape != img2.shape:
        print('binary_image_similarity: Different image size')
        return 0.0
    xor_img = cv2.bitwise_xor(img1, img2)
    return 1.0 - np.float(np.count_nonzero(xor_img)) / (img1.shape[0]*img2.shape[1])

# src, dst: contour_dataset or template_dataset (holding member variables box, solid)
def get_transform_by_rotated_rectangle(src, dst):
    # Rotated patterns are created when starting index is slided
    dst_box2 = np.vstack([dst.box, dst.box])
    max_similarity = 0.0
    max_converted_img = np.zeros((dst.solid.shape[1], dst.solid.shape[0]), 'uint8')
    for i in range(4):
        M = cv2.getAffineTransform(src.box[0:3], dst_box2[i:i+3])
        converted_img = cv2.warpAffine(src.solid, M, dsize=(dst.solid.shape[1], dst.solid.shape[0]), flags=cv2.INTER_NEAREST)
        similarity = binary_image_similarity(converted_img, dst.solid)
        if similarity > max_similarity:
            M_rtn = M
            max_similarity = similarity
            max_converted_img = converted_img
    return M_rtn, max_similarity, max_converted_img

def get_similarity_with_template(target_data, template_data, sim_th_high=0.95, sim_th_low=0.7):
    _,(w1,h1), _ = target_data.rrect
    _,(w2,h2), _ = template_data.rrect
    r = w1/h1 if w1 < h1 else h1/w1
    r = r * h2/w2 if w2 < h2 else r * w2/h2
    M, sim_init, _ = get_transform_by_rotated_rectangle(template_data, target_data)
    if sim_init > sim_th_high or sim_init < sim_th_low or r > 1.4 or r < 0.7:
        dsize = (template_data.solid.shape[1], template_data.solid.shape[0])
        flags = cv2.INTER_NEAREST|cv2.WARP_INVERSE_MAP
        converted_img = cv2.warpAffine(target_data.solid, M, dsize=dsize, flags=flags)
        return sim_init, converted_img
    M, _ = icp(template_data.pts, target_data.pts, initial_matrix=M)
    Minv = cv2.invertAffineTransform(M)
    converted_ctr = np.zeros_like(target_data.ctr)
    for i in range(target_data.ctr.shape[0]):
        converted_ctr[i,0,:] = (Minv[:,0:2] @ target_data.ctr[i,0,:]) + Minv[:,2]
    converted_img = create_solid_contour(converted_ctr, img_shape=template_data.solid.shape)
    val = binary_image_similarity(converted_img, template_data.solid)
    return val, converted_img

def get_similarity_with_template_zero(target_data, template_data):
    dsize = (template_data.solid.shape[1], template_data.solid.shape[0])
    converted_img = cv2.resize(target_data.solid, dsize=dsize, interpolation=cv2.INTER_NEAREST)
    val = binary_image_similarity(converted_img, template_data.solid)
    return val, converted_img

def get_similarities(target, templates):
    similarities = []
    converted_imgs = []
    for tmpl in templates:
        if tmpl.num == 0:
            sim,converted_img = get_similarity_with_template_zero(target, tmpl)
        else:
            sim,converted_img = get_similarity_with_template(target, tmpl)
        similarities += [sim]
        converted_imgs += [converted_img]
    return similarities, converted_imgs

# target: Single contour to compare
# templates: List of template_dataset (for numbers 0, 1, 2, 3, 5)
# svm: Trained SVM
# return: determined number (0,1,2,3,5), -1 if none corresponds
def determine_number(target, templates, svm):
    similarities,_ = get_similarities(target, templates)
    _, result = svm.predict(np.array(similarities))
    return int(result[0])

SVM学習関連処理

学習用データのサンプルでは、同じ状況の再現のため、乱数のシードを指定できるようにしました。

def get_random_sample(data_in, labels_in, selected_labels, n_samples, seed=None):
    random.seed(seed)
    data_rtn = []
    labels_rtn = []
    for lab in selected_labels:
        samples = [d for i,d in enumerate(data_in) if labels_in[i]==lab]
        n = min(n_samples, len(samples))
        data_rtn += random.sample(samples, n)
        labels_rtn += [lab] * n
    return data_rtn, labels_rtn

def prepare_svm(train_data, train_labels):
    svm = cv2.ml.SVM_create()
    svm.setKernel(cv2.ml.SVM_LINEAR)
    svm.setType(cv2.ml.SVM_C_SVC)
    svm.setC(100)
    svm.setGamma(1)
    svm.train(np.array(train_data, 'float32'), cv2.ml.ROW_SAMPLE, np.array(train_labels))
    return svm

def print_stat(svm_results, svm_labels):
    stats = {k:{k2:0 for k2 in [-1, 0, 1, 2, 3, 5]} for k in [-1, 0, 1, 2, 3, 5]}
    for res, lab in zip(svm_results[1], svm_labels):
        stats[lab][int(res[0])] += 1
    for k,v in stats.items():
        print('label {:>2}'.format(k), ': {', end='')
        for k2,v2 in v.items():
            print('{}: {:>2}, '.format(k2,v2), end='')
        print('}')

def print_similarity_vector(sim, end=''):
    print('[',end='')
    for s in sim: print('{:.3f}, '.format(s), end='')
    print(']', end=end)

データの用意

ここから、実データからの必要なデータ取り出しを行っていきます。
一部のデータは、後で再利用のために保存します。

輪郭データの取得

original_imgs = [img1, img2, img3, img4, img5, img6, img7]
resized_imgs = []
resized_ctrs = []
original_img_idx = []
subctrs_all = []
subimgs_all = []
for idx, original_img in enumerate(original_imgs):
    ctrs, img = detect_candidate_contours(original_img)
    resized_imgs += [img]
    resized_ctrs += [ctrs]
    
    for ctr in ctrs:
        original_img_idx += [idx]
        subimg,subctr = create_contour_area_image(img, ctr)
        subctrs_all += [subctr]
        subimgs_all += [subimg]

テンプレートデータの用意

今までやっていた通り、3つ目の画像(2020年)、5つ目の画像(2021年)では、"3点"のデータがないので、1つ目の画像(2019年)のデータで代用します。

ctrs1_idx_zero = 26
ctrs1_idx_one = 27
ctrs1_idx_two = 24
ctrs1_idx_three = 33
ctrs1_idx_five = 8
ctrs1_idx_numbers = [ctrs1_idx_zero, ctrs1_idx_one, ctrs1_idx_two, ctrs1_idx_three, ctrs1_idx_five]

subimgs1 = []
subctrs1 = []
binimgs1 = []
subctrs1_selected_pts = []
for i,idx in enumerate(ctrs1_idx_numbers):
    img, ctr = create_contour_area_image(resized_imgs[0], resized_ctrs[0][idx])
    binimg, M, ctr2 = create_upright_solid_contour(ctr)
    img2 = cv2.warpAffine(img.copy(), M, (binimg.shape[1], binimg.shape[0]))
    subimgs1 += [img2]
    subctrs1 += [ctr2]
    binimgs1 += [binimg]
    ctr_selected_pts = [j for j in range(ctr2.shape[0]) if j % 5 == 0]
    if i != 0:
        subctrs1_selected_pts += [ctr_selected_pts]
    ctr_img = cv2.drawContours(img2.copy(), [ctr2], -1, (0,255,0), 2)
    pts_img = img2.copy()
    for p in ctr_selected_pts:
        pts_img = cv2.drawMarker(pts_img, ctr2[p,0,:], (0,255,0), markerType=cv2.MARKER_CROSS, markerSize=3)
    plt.subplot(3,5,1+i), plt.imshow(cv2.cvtColor(ctr_img, cv2.COLOR_BGR2RGB)), plt.xticks([]), plt.yticks([])
    plt.subplot(3,5,6+i), plt.imshow(binimg,cmap='gray'), plt.xticks([]), plt.yticks([])
    plt.subplot(3,5,11+i), plt.imshow(cv2.cvtColor(pts_img, cv2.COLOR_BGR2RGB), cmap='gray'), plt.xticks([]), plt.yticks([])
plt.show()

ctrs3_idx_zero = 7
ctrs3_idx_one = 4
ctrs3_idx_two = 17
ctrs3_idx_five = 6
ctrs3_idx_numbers = [ctrs3_idx_zero, ctrs3_idx_one, ctrs3_idx_two, ctrs3_idx_five]

subimgs3 = []
subctrs3 = []
binimgs3 = []
subctrs3_selected_pts = []
for i,idx in enumerate(ctrs3_idx_numbers):
    img, ctr = create_contour_area_image(resized_imgs[2], resized_ctrs[2][idx])
    binimg, M, ctr2 = create_upright_solid_contour(ctr)
    img2 = cv2.warpAffine(img.copy(), M, (binimg.shape[1], binimg.shape[0]))
    subimgs3 += [img2]
    subctrs3 += [ctr2]
    binimgs3 += [binimg]
    ctr_selected_pts = [j for j in range(ctr2.shape[0]) if j% 5 == 0]
    if i != 0:
        subctrs3_selected_pts += [ctr_selected_pts]
    ctr_img = cv2.drawContours(img2.copy(), [ctr2], -1, (0,255,0), 2)
    pts_img = img2.copy()
    for p in ctr_selected_pts:
        pts_img = cv2.drawMarker(pts_img, ctr2[p,0,:], (0,255,0), markerType=cv2.MARKER_CROSS, markerSize=3)
    plt.subplot(3,4,1+i), plt.imshow(cv2.cvtColor(ctr_img, cv2.COLOR_BGR2RGB)), plt.xticks([]), plt.yticks([])
    plt.subplot(3,4,5+i), plt.imshow(binimg,cmap='gray'), plt.xticks([]), plt.yticks([])
    plt.subplot(3,4,9+i), plt.imshow(cv2.cvtColor(pts_img, cv2.COLOR_BGR2RGB)), plt.xticks([]), plt.yticks([])
plt.show()

subimgs3.insert(3, subimgs1[3])
subctrs3.insert(3, subctrs1[3])
binimgs3.insert(3, binimgs1[3])
subctrs3_selected_pts.insert(2, subctrs1_selected_pts[2])

ctrs5_idx_zero = 3
ctrs5_idx_one = 4
ctrs5_idx_two = 2
ctrs5_idx_five = 5
ctrs5_idx_numbers = [ctrs5_idx_zero, ctrs5_idx_one, ctrs5_idx_two, ctrs5_idx_five]

subimgs5 = []
subctrs5 = []
binimgs5 = []
subctrs5_selected_pts = []
for i,idx in enumerate(ctrs5_idx_numbers):
    img, ctr = create_contour_area_image(resized_imgs[4], resized_ctrs[4][idx])
    binimg, M, ctr2 = create_upright_solid_contour(ctr)
    img2 = cv2.warpAffine(img.copy(), M, (binimg.shape[1], binimg.shape[0]))
    subimgs5 += [img2]
    subctrs5 += [ctr2]
    binimgs5 += [binimg]
    ctr_selected_pts = [j for j in range(ctr2.shape[0]) if j % 5 == 0]
    if i != 0:
        subctrs5_selected_pts += [ctr_selected_pts]
    ctr_img = cv2.drawContours(img2.copy(), [ctr2], -1, (0,255,0), 2)
    pts_img = img2.copy()
    for p in ctr_selected_pts:
        pts_img = cv2.drawMarker(pts_img, ctr2[p,0,:], (0,255,0), markerType=cv2.MARKER_CROSS, markerSize=3)
    plt.subplot(3,4,1+i), plt.imshow(cv2.cvtColor(ctr_img, cv2.COLOR_BGR2RGB)), plt.xticks([]), plt.yticks([])
    plt.subplot(3,4,5+i), plt.imshow(binimg,cmap='gray'), plt.xticks([]), plt.yticks([])
    plt.subplot(3,4,9+i), plt.imshow(cv2.cvtColor(pts_img, cv2.COLOR_BGR2RGB)), plt.xticks([]), plt.yticks([])
plt.show()

subimgs5.insert(3, subimgs1[3])
subctrs5.insert(3, subctrs1[3])
binimgs5.insert(3, binimgs1[3])
subctrs5_selected_pts.insert(2, subctrs1_selected_pts[2])

正解ラベル

labels1 = [-1,-1,-1,-1,-1
           ,-1,5,0,5,1
           ,5,0,2,1,2
           ,-1,-1,1,1,5
           ,0,2,5,0,2
           ,5,0,1,2,-1
           ,5,1,2,3,1
           ,5,0,-1]

labels2 = [-1,-1,-1,-1,-1
           ,-1,5,0,5,1
           ,5,0,2,1,2
           ,-1,-1,1,1,5
           ,0,-1,2,5,0
           ,2,5,0,1,2
           ,5,0,1,-1,2
           ,-1,-1,-1,3,-1
           ,5,0,-1,1,-1
           ,-1]

labels3 = [-1,-1,-1,-1,1
           ,1,5,0,1,1
           ,5,0,5,0,-1
           ,-1,-1,2,-1,-1
           ,-1,1,1,1,-1
           ,1,-1,-1,1,1
           ,-1,2,-1,1,-1
           ,1,2,-1,1,-1
           ,-1,2,5,-1,0
           ,-1,1,1]

labels4 = [-1,-1,-1,-1,-1
           ,-1,-1,-1,-1,-1
           ,-1,-1,-1,-1,-1
           ,-1,-1,-1,1,1
           ,1,1,1,1,1
           ,-1,5,0,2,5
           ,0,2,1,2,2
           ,-1,-1,-1,1,1
           ,1]

labels5 = [-1,-1,2,0,1
           ,5,-1,1,1,1
           ,1,1,1,1,1
           ,1,-1,5,1,0
           ,5,1,2,0,5
           ,0,2,1,2,2
           ,-1,-1,1,1,1
           ]

labels6 = [-1,0,1,5,2
            ,-1,1,1,1,1
            ,5,1,0,5,0
            ,2,1,5,0,2
            ,2,2,1,-1,-1
            ,1,1,1,1,1
            ,1,1,1]

labels7 = [-1,-1,-1,-1,-1
           ,-1,1,2,2,2
           ,2,1,2,2,2
           ,1,-1,-1,-1,2
           ,1,2,1,1]

labels_all = labels1 + labels2 + labels3 + labels4 + labels5 + labels6 + labels7

データセットの作成

# Prepare template data for "0"
templates1 = [template_dataset(subctrs1[0], 0)]
templates3 = [template_dataset(subctrs3[0], 0)]
templates5 = [template_dataset(subctrs5[0], 0)]
# Prepare template data for other numbers
numbers = [1, 2, 3, 5]
for i,num in enumerate(numbers):
    templates1 += [template_dataset(subctrs1[i+1], num, subctrs1_selected_pts[i])]
    templates3 += [template_dataset(subctrs3[i+1], num, subctrs3_selected_pts[i])]
    templates5 += [template_dataset(subctrs5[i+1], num, subctrs5_selected_pts[i])]
ctr_datasets_all = [contour_dataset(ctr) for ctr in subctrs_all]

一致度計算実施

templates_sel = [1,1,3,5,5,5,5]
def select_template(i):
    img_idx = original_img_idx[i]
    if templates_sel[img_idx] == 1:
        return templates1
    elif templates_sel[img_idx] == 3:
        return templates3
    elif templates_sel[img_idx] == 5:
        return templates5
    else:
        return templates1

similarities_all = []
converted_imgs_all = []
print('  Contour No. ', end='')
for i,target_ctr in enumerate(ctr_datasets_all):
    templates = select_template(i)
    print(i, ' ', end='')
    sims, imgs = get_similarities(target_ctr, templates)
    similarities_all += [sims]
    converted_imgs_all += [imgs]

  Contour No. 0  1  2  3  4  5  6  7  8  9  10  11  12  13  14  15  16  17  18  19  20  21  22  23  24  25  26  27  28  29  30  31  32  33  34  35  36  37  38  39  40  41  42  43  44  45  46  47  48  49  50  51  52  53  54  55  56  57  58  59  60  61  62  63  64  65  66  67  68  69  70  71  72  73  74  75  76  77  78  79  80  81  82  83  84  85  86  87  88  89  90  91  92  93  94  95  96  97  98  99  100  101  102  103  104  105  106  107  108  109  110  111  112  113  114  115  116  117  118  119  120  121  122  123  124  125  126  127  128  129  130  131  132  133  134  135  136  137  138  139  140  141  142  143  144  145  146  147  148  149  150  151  152  153  154  155  156  157  158  159  160  161  162  163  164  165  166  167  168  169  170  171  172  173  174  175  176  177  178  179  180  181  182  183  184  185  186  187  188  189  190  191  192  193  194  195  196  197  198  199  200  201  202  203  204  205  206  207  208  209  210  211  212  213  214  215  216  217  218  219  220  221  222  223  224  225  226  227  228  229  230  231  232  233  234  235  236  237  238  239  240  241  242  243  244  245  246  247  248  249  250  251  252  253  254  255  256  257  258  259  260  261  262  263  264  

得られた一致度を全て表示しておきます。
長くなっちゃいますが…

for sim, lab in zip(similarities_all, labels_all):
    print('label {:>2}'.format(lab), ': ', end='')
    print_similarity_vector(sim, end='\n')

label -1 : [0.825, 0.879, 0.666, 0.649, 0.710, ]
label -1 : [0.820, 0.783, 0.658, 0.667, 0.699, ]
label -1 : [0.841, 0.757, 0.684, 0.676, 0.733, ]
label -1 : [0.816, 0.730, 0.676, 0.699, 0.731, ]
label -1 : [0.844, 0.765, 0.691, 0.708, 0.738, ]
label -1 : [0.860, 0.746, 0.697, 0.681, 0.736, ]
label  5 : [0.717, 0.816, 0.685, 0.812, 0.919, ]
label  0 : [0.868, 0.836, 0.608, 0.679, 0.731, ]
label  5 : [0.746, 0.748, 0.672, 0.792, 0.950, ]
label  1 : [0.729, 0.942, 0.766, 0.743, 0.731, ]
label  5 : [0.731, 0.774, 0.668, 0.781, 0.921, ]
label  0 : [0.942, 0.762, 0.695, 0.669, 0.728, ]
label  2 : [0.660, 0.753, 0.937, 0.754, 0.724, ]
label  1 : [0.744, 0.947, 0.710, 0.702, 0.686, ]
label  2 : [0.650, 0.777, 0.957, 0.754, 0.702, ]
label -1 : [0.682, 0.747, 0.643, 0.630, 0.647, ]
label -1 : [0.796, 0.672, 0.715, 0.783, 0.878, ]
label  1 : [0.709, 0.952, 0.801, 0.784, 0.755, ]
label  1 : [0.625, 0.955, 0.834, 0.807, 0.805, ]
label  5 : [0.732, 0.759, 0.663, 0.786, 0.918, ]
label  0 : [0.967, 0.722, 0.670, 0.659, 0.732, ]
label  2 : [0.665, 0.715, 0.945, 0.751, 0.682, ]
label  5 : [0.719, 0.680, 0.691, 0.801, 0.898, ]
label  0 : [0.963, 0.722, 0.678, 0.664, 0.728, ]
label  2 : [0.670, 0.732, 0.963, 0.754, 0.697, ]
label  5 : [0.750, 0.693, 0.661, 0.674, 0.913, ]
label  0 : [0.895, 0.734, 0.684, 0.629, 0.752, ]
label  1 : [0.744, 0.954, 0.740, 0.708, 0.706, ]
label  2 : [0.681, 0.720, 0.942, 0.750, 0.692, ]
label -1 : [0.820, 0.738, 0.691, 0.713, 0.750, ]
label  5 : [0.671, 0.761, 0.653, 0.610, 0.784, ]
label  1 : [0.699, 0.953, 0.711, 0.688, 0.718, ]
label  2 : [0.621, 0.812, 0.958, 0.745, 0.736, ]
label  3 : [0.701, 0.664, 0.773, 1.000, 0.669, ]
label  1 : [0.542, 0.959, 0.837, 0.809, 0.818, ]
label  5 : [0.737, 0.728, 0.704, 0.782, 0.924, ]
label  0 : [0.865, 0.806, 0.655, 0.669, 0.721, ]
label -1 : [0.690, 0.781, 0.546, 0.677, 0.695, ]
label -1 : [0.840, 0.879, 0.677, 0.662, 0.765, ]
label -1 : [0.827, 0.787, 0.696, 0.688, 0.737, ]
label -1 : [0.844, 0.770, 0.645, 0.645, 0.705, ]
label -1 : [0.829, 0.750, 0.676, 0.751, 0.701, ]
label -1 : [0.862, 0.735, 0.684, 0.691, 0.728, ]
label -1 : [0.855, 0.758, 0.664, 0.664, 0.732, ]
label  5 : [0.709, 0.806, 0.687, 0.792, 0.901, ]
label  0 : [0.879, 0.821, 0.640, 0.667, 0.724, ]
label  5 : [0.734, 0.762, 0.677, 0.706, 0.928, ]
label  1 : [0.728, 0.941, 0.771, 0.757, 0.753, ]
label  5 : [0.726, 0.777, 0.675, 0.730, 0.931, ]
label  0 : [0.932, 0.782, 0.628, 0.660, 0.724, ]
label  2 : [0.657, 0.751, 0.946, 0.752, 0.731, ]
label  1 : [0.748, 0.936, 0.737, 0.708, 0.705, ]
label  2 : [0.636, 0.782, 0.956, 0.758, 0.714, ]
label -1 : [0.780, 0.648, 0.735, 0.787, 0.825, ]
label -1 : [0.687, 0.727, 0.642, 0.646, 0.660, ]
label  1 : [0.708, 0.943, 0.796, 0.778, 0.749, ]
label  1 : [0.618, 0.952, 0.836, 0.811, 0.812, ]
label  5 : [0.734, 0.775, 0.682, 0.785, 0.920, ]
label  0 : [0.962, 0.708, 0.666, 0.664, 0.727, ]
label -1 : [0.778, 0.783, 0.687, 0.701, 0.737, ]
label  2 : [0.663, 0.722, 0.943, 0.748, 0.681, ]
label  5 : [0.739, 0.666, 0.692, 0.812, 0.904, ]
label  0 : [0.962, 0.704, 0.672, 0.665, 0.724, ]
label  2 : [0.668, 0.736, 0.951, 0.746, 0.700, ]
label  5 : [0.750, 0.693, 0.653, 0.675, 0.922, ]
label  0 : [0.905, 0.707, 0.662, 0.630, 0.752, ]
label  1 : [0.745, 0.940, 0.738, 0.712, 0.701, ]
label  2 : [0.685, 0.718, 0.946, 0.755, 0.680, ]
label  5 : [0.750, 0.675, 0.710, 0.794, 0.915, ]
label  0 : [0.827, 0.785, 0.669, 0.627, 0.743, ]
label  1 : [0.695, 0.945, 0.717, 0.692, 0.705, ]
label -1 : [0.775, 0.859, 0.708, 0.691, 0.725, ]
label  2 : [0.610, 0.814, 0.959, 0.748, 0.742, ]
label -1 : [0.798, 0.863, 0.684, 0.697, 0.788, ]
label -1 : [0.805, 0.851, 0.696, 0.675, 0.733, ]
label -1 : [0.786, 0.765, 0.675, 0.659, 0.686, ]
label  3 : [0.708, 0.671, 0.767, 0.969, 0.668, ]
label -1 : [0.942, 0.719, 0.693, 0.695, 0.736, ]
label  5 : [0.750, 0.737, 0.693, 0.781, 0.897, ]
label  0 : [0.845, 0.789, 0.660, 0.674, 0.745, ]
label -1 : [0.802, 0.805, 0.702, 0.701, 0.749, ]
label  1 : [0.533, 0.944, 0.835, 0.813, 0.818, ]
label -1 : [0.796, 0.821, 0.687, 0.696, 0.698, ]
label -1 : [0.776, 0.824, 0.753, 0.714, 0.738, ]
label -1 : [0.824, 0.699, 0.615, 0.639, 0.693, ]
label -1 : [0.694, 0.682, 0.633, 0.664, 0.656, ]
label -1 : [0.868, 0.735, 0.673, 0.691, 0.761, ]
label -1 : [0.825, 0.735, 0.673, 0.680, 0.761, ]
label  1 : [0.691, 0.954, 0.762, 0.784, 0.729, ]
label  1 : [0.668, 0.947, 0.743, 0.743, 0.731, ]
label  5 : [0.765, 0.711, 0.683, 0.706, 0.938, ]
label  0 : [1.000, 0.710, 0.650, 0.628, 0.756, ]
label  1 : [0.646, 0.947, 0.714, 0.707, 0.692, ]
label  1 : [0.666, 0.945, 0.743, 0.732, 0.731, ]
label  5 : [0.679, 0.768, 0.687, 0.752, 0.929, ]
label  0 : [0.843, 0.753, 0.688, 0.635, 0.697, ]
label  5 : [0.759, 0.717, 0.690, 0.681, 0.934, ]
label  0 : [0.956, 0.690, 0.633, 0.625, 0.690, ]
label -1 : [0.825, 0.811, 0.735, 0.761, 0.692, ]
label -1 : [0.811, 0.700, 0.636, 0.668, 0.719, ]
label -1 : [0.793, 0.667, 0.632, 0.641, 0.730, ]
label  2 : [0.650, 0.785, 0.979, 0.762, 0.705, ]
label -1 : [0.784, 0.751, 0.686, 0.691, 0.701, ]
label -1 : [0.847, 0.729, 0.668, 0.679, 0.733, ]
label -1 : [0.808, 0.697, 0.650, 0.660, 0.719, ]
label  1 : [0.741, 0.940, 0.738, 0.715, 0.674, ]
label  1 : [0.671, 0.944, 0.730, 0.717, 0.691, ]
label  1 : [0.670, 0.941, 0.730, 0.721, 0.698, ]
label -1 : [0.797, 0.747, 0.689, 0.692, 0.734, ]
label  1 : [0.685, 0.945, 0.718, 0.695, 0.685, ]
label -1 : [0.822, 0.713, 0.644, 0.692, 0.706, ]
label -1 : [0.814, 0.667, 0.624, 0.650, 0.726, ]
label  1 : [0.706, 0.952, 0.716, 0.713, 0.690, ]
label  1 : [0.639, 0.954, 0.687, 0.689, 0.674, ]
label -1 : [0.798, 0.671, 0.620, 0.654, 0.749, ]
label  2 : [0.662, 0.711, 0.951, 0.753, 0.651, ]
label -1 : [0.811, 0.718, 0.688, 0.723, 0.721, ]
label  1 : [0.639, 0.960, 0.683, 0.690, 0.674, ]
label -1 : [0.802, 0.692, 0.615, 0.658, 0.731, ]
label  1 : [0.702, 0.948, 0.726, 0.715, 0.678, ]
label  2 : [0.601, 0.755, 0.944, 0.754, 0.676, ]
label -1 : [0.819, 0.714, 0.647, 0.730, 0.719, ]
label  1 : [0.677, 0.941, 0.718, 0.706, 0.692, ]
label -1 : [0.823, 0.684, 0.640, 0.672, 0.744, ]
label -1 : [0.648, 0.847, 0.697, 0.666, 0.681, ]
label  2 : [0.592, 0.757, 0.965, 0.757, 0.697, ]
label  5 : [0.753, 0.692, 0.683, 0.684, 0.925, ]
label -1 : [0.648, 0.762, 0.662, 0.664, 0.762, ]
label  0 : [0.956, 0.696, 0.640, 0.629, 0.703, ]
label -1 : [0.811, 0.688, 0.637, 0.671, 0.753, ]
label  1 : [0.678, 0.944, 0.702, 0.689, 0.690, ]
label  1 : [0.656, 0.949, 0.684, 0.703, 0.679, ]
label -1 : [0.561, 0.841, 0.807, 0.802, 0.845, ]
label -1 : [0.791, 0.805, 0.721, 0.730, 0.730, ]
label -1 : [0.846, 0.798, 0.613, 0.641, 0.669, ]
label -1 : [0.797, 0.786, 0.683, 0.681, 0.721, ]
label -1 : [0.849, 0.748, 0.700, 0.676, 0.739, ]
label -1 : [0.867, 0.731, 0.655, 0.648, 0.766, ]
label -1 : [0.874, 0.761, 0.672, 0.641, 0.717, ]
label -1 : [0.833, 0.817, 0.648, 0.674, 0.684, ]
label -1 : [0.908, 0.825, 0.694, 0.703, 0.659, ]
label -1 : [0.950, 0.822, 0.667, 0.693, 0.710, ]
label -1 : [0.894, 0.812, 0.611, 0.679, 0.680, ]
label -1 : [0.854, 0.730, 0.641, 0.644, 0.737, ]
label -1 : [0.894, 0.801, 0.696, 0.696, 0.714, ]
label -1 : [0.806, 0.769, 0.694, 0.658, 0.679, ]
label -1 : [0.908, 0.816, 0.677, 0.696, 0.766, ]
label -1 : [0.700, 0.706, 0.656, 0.650, 0.686, ]
label -1 : [0.804, 0.709, 0.707, 0.770, 0.848, ]
label -1 : [0.749, 0.721, 0.664, 0.695, 0.740, ]
label  1 : [0.706, 0.948, 0.771, 0.721, 0.749, ]
label  1 : [0.684, 0.959, 0.769, 0.728, 0.740, ]
label  1 : [0.658, 0.955, 0.807, 0.763, 0.778, ]
label  1 : [0.641, 0.958, 0.837, 0.798, 0.790, ]
label  1 : [0.512, 0.966, 0.874, 0.829, 0.841, ]
label  1 : [0.696, 0.948, 0.776, 0.726, 0.749, ]
label  1 : [0.684, 0.950, 0.773, 0.722, 0.742, ]
label -1 : [0.502, 0.738, 0.779, 0.647, 0.712, ]
label  5 : [0.731, 0.688, 0.678, 0.792, 0.920, ]
label  0 : [0.931, 0.718, 0.646, 0.630, 0.701, ]
label  2 : [0.652, 0.753, 0.936, 0.755, 0.659, ]
label  5 : [0.738, 0.712, 0.653, 0.688, 0.913, ]
label  0 : [0.904, 0.825, 0.639, 0.635, 0.716, ]
label  2 : [0.653, 0.744, 0.935, 0.749, 0.660, ]
label  1 : [0.639, 0.950, 0.806, 0.758, 0.779, ]
label  2 : [0.640, 0.760, 0.930, 0.750, 0.674, ]
label  2 : [0.632, 0.763, 0.933, 0.751, 0.670, ]
label -1 : [0.639, 0.866, 0.668, 0.667, 0.713, ]
label -1 : [0.593, 0.783, 0.596, 0.620, 0.762, ]
label -1 : [0.524, 0.772, 0.683, 0.692, 0.714, ]
label  1 : [0.698, 0.948, 0.756, 0.710, 0.746, ]
label  1 : [0.765, 0.947, 0.749, 0.705, 0.716, ]
label  1 : [0.667, 0.950, 0.791, 0.732, 0.762, ]
label -1 : [0.826, 0.729, 0.671, 0.645, 0.703, ]
label -1 : [0.827, 0.738, 0.665, 0.650, 0.723, ]
label  2 : [0.628, 0.794, 0.939, 0.756, 0.670, ]
label  0 : [0.812, 0.814, 0.643, 0.620, 0.702, ]
label  1 : [0.556, 0.970, 0.867, 0.825, 0.738, ]
label  5 : [0.639, 0.775, 0.670, 0.784, 0.964, ]
label -1 : [0.473, 0.880, 0.593, 0.620, 0.661, ]
label  1 : [0.659, 0.959, 0.840, 0.745, 0.738, ]
label  1 : [0.770, 0.939, 0.776, 0.718, 0.738, ]
label  1 : [0.704, 0.951, 0.778, 0.726, 0.737, ]
label  1 : [0.713, 0.949, 0.784, 0.748, 0.733, ]
label  1 : [0.637, 0.951, 0.816, 0.756, 0.723, ]
label  1 : [0.701, 0.945, 0.831, 0.750, 0.752, ]
label  1 : [0.511, 0.955, 0.867, 0.829, 0.835, ]
label  1 : [0.708, 0.930, 0.788, 0.743, 0.734, ]
label  1 : [0.674, 0.951, 0.804, 0.724, 0.743, ]
label -1 : [0.909, 0.806, 0.682, 0.671, 0.690, ]
label  5 : [0.624, 0.674, 0.638, 0.662, 0.919, ]
label  1 : [0.636, 0.946, 0.806, 0.763, 0.733, ]
label  0 : [0.840, 0.820, 0.620, 0.627, 0.713, ]
label  5 : [0.710, 0.685, 0.683, 0.681, 0.912, ]
label  1 : [0.722, 0.942, 0.823, 0.780, 0.737, ]
label  2 : [0.633, 0.760, 0.939, 0.767, 0.675, ]
label  0 : [0.925, 0.727, 0.645, 0.649, 0.690, ]
label  5 : [0.780, 0.722, 0.641, 0.661, 0.931, ]
label  0 : [0.932, 0.750, 0.656, 0.633, 0.716, ]
label  2 : [0.653, 0.723, 0.938, 0.762, 0.641, ]
label  1 : [0.667, 0.917, 0.815, 0.761, 0.734, ]
label  2 : [0.693, 0.757, 0.922, 0.744, 0.687, ]
label  2 : [0.675, 0.738, 0.931, 0.753, 0.655, ]
label -1 : [0.581, 0.792, 0.594, 0.588, 0.699, ]
label -1 : [0.578, 0.823, 0.707, 0.668, 0.700, ]
label  1 : [0.704, 0.957, 0.713, 0.668, 0.743, ]
label  1 : [0.784, 0.927, 0.759, 0.718, 0.705, ]
label  1 : [0.728, 0.948, 0.756, 0.704, 0.751, ]
label -1 : [0.748, 0.825, 0.723, 0.730, 0.782, ]
label  0 : [0.887, 0.816, 0.638, 0.672, 0.681, ]
label  1 : [0.506, 0.960, 0.864, 0.842, 0.852, ]
label  5 : [0.704, 0.795, 0.679, 0.799, 0.926, ]
label  2 : [0.588, 0.763, 0.923, 0.752, 0.676, ]
label -1 : [0.507, 0.676, 0.625, 0.583, 0.567, ]
label  1 : [0.563, 0.922, 0.856, 0.820, 0.832, ]
label  1 : [0.727, 0.938, 0.820, 0.791, 0.779, ]
label  1 : [0.748, 0.937, 0.736, 0.714, 0.754, ]
label  1 : [0.751, 0.937, 0.763, 0.728, 0.775, ]
label  5 : [0.608, 0.700, 0.751, 0.711, 0.911, ]
label  1 : [0.729, 0.879, 0.789, 0.756, 0.684, ]
label  0 : [0.786, 0.820, 0.623, 0.615, 0.703, ]
label  5 : [0.677, 0.712, 0.638, 0.816, 0.885, ]
label  0 : [0.881, 0.804, 0.612, 0.624, 0.705, ]
label  2 : [0.615, 0.789, 0.944, 0.771, 0.665, ]
label  1 : [0.605, 0.951, 0.846, 0.807, 0.809, ]
label  5 : [0.706, 0.694, 0.668, 0.698, 0.912, ]
label  0 : [0.929, 0.736, 0.664, 0.646, 0.695, ]
label  2 : [0.625, 0.736, 0.918, 0.763, 0.659, ]
label  2 : [0.674, 0.736, 0.919, 0.777, 0.657, ]
label  2 : [0.672, 0.740, 0.930, 0.746, 0.667, ]
label  1 : [0.706, 0.941, 0.765, 0.729, 0.751, ]
label -1 : [0.566, 0.826, 0.604, 0.581, 0.680, ]
label -1 : [0.587, 0.828, 0.695, 0.674, 0.676, ]
label  1 : [0.758, 0.884, 0.736, 0.702, 0.679, ]
label  1 : [0.784, 0.925, 0.756, 0.736, 0.742, ]
label  1 : [0.750, 0.933, 0.714, 0.690, 0.746, ]
label  1 : [0.709, 0.925, 0.799, 0.764, 0.711, ]
label  1 : [0.567, 0.959, 0.852, 0.809, 0.740, ]
label  1 : [0.578, 0.938, 0.850, 0.814, 0.822, ]
label  1 : [0.767, 0.938, 0.717, 0.696, 0.772, ]
label  1 : [0.727, 0.935, 0.768, 0.739, 0.744, ]
label -1 : [0.851, 0.864, 0.698, 0.668, 0.646, ]
label -1 : [0.811, 0.783, 0.728, 0.724, 0.762, ]
label -1 : [0.960, 0.817, 0.667, 0.725, 0.724, ]
label -1 : [0.818, 0.827, 0.724, 0.692, 0.776, ]
label -1 : [0.796, 0.750, 0.694, 0.659, 0.664, ]
label -1 : [0.872, 0.776, 0.711, 0.697, 0.678, ]
label  1 : [0.755, 0.939, 0.801, 0.753, 0.738, ]
label  2 : [0.664, 0.759, 0.933, 0.752, 0.686, ]
label  2 : [0.645, 0.762, 0.933, 0.756, 0.683, ]
label  2 : [0.669, 0.760, 0.934, 0.752, 0.686, ]
label  2 : [0.640, 0.794, 0.932, 0.764, 0.669, ]
label  1 : [0.726, 0.946, 0.722, 0.670, 0.716, ]
label  2 : [0.654, 0.742, 0.931, 0.767, 0.680, ]
label  2 : [0.606, 0.841, 0.929, 0.750, 0.658, ]
label  2 : [0.587, 0.841, 0.922, 0.766, 0.691, ]
label  1 : [0.680, 0.941, 0.838, 0.783, 0.784, ]
label -1 : [0.557, 0.829, 0.675, 0.598, 0.679, ]
label -1 : [0.509, 0.771, 0.691, 0.645, 0.692, ]
label -1 : [0.518, 0.814, 0.723, 0.639, 0.694, ]
label  2 : [0.637, 0.800, 0.935, 0.765, 0.670, ]
label  1 : [0.736, 0.926, 0.743, 0.700, 0.743, ]
label  2 : [0.658, 0.780, 0.949, 0.760, 0.672, ]
label  1 : [0.738, 0.936, 0.825, 0.778, 0.768, ]
label  1 : [0.728, 0.938, 0.718, 0.671, 0.730, ]

SVM学習実施

SVMの学習を実施します。
結果が良ければ、学習したSVMモデルを保存、今後使用します。
30番の輪郭は輪郭検出がうまくいっていないのが分かっているので削除しておきます。

svm_inputs = copy.deepcopy(similarities_all)
svm_labels = copy.deepcopy(labels_all)

# Remove inadequate contour data in img1
del svm_inputs[30]
del svm_labels[30]

train_data, train_labels = get_random_sample(svm_inputs, svm_labels, [-1,0,1,2,3,5], 20, seed=123)
svm = prepare_svm(train_data, train_labels)

SVM推論実施

学習したSVMモデルの性能を確認。

result = svm.predict(np.array(svm_inputs, 'float32'))
print_stat(result, svm_labels)

label -1 : {-1: 73, 0: 13, 1:  0, 2:  0, 3:  0, 5:  3, }
label  0 : {-1:  5, 0: 22, 1:  0, 2:  0, 3:  0, 5:  0, }
label  1 : {-1:  0, 0:  0, 1: 78, 2:  0, 3:  0, 5:  0, }
label  2 : {-1:  0, 0:  0, 1:  0, 2: 39, 3:  0, 5:  0, }
label  3 : {-1:  0, 0:  0, 1:  0, 2:  0, 3:  2, 5:  0, }
label  5 : {-1:  0, 0:  0, 1:  0, 2:  0, 3:  0, 5: 29, }

"0"以外の数字の輪郭であれば全て正解していますが、非数字の輪郭で判定の失敗がいくつかあります。
判定に失敗した輪郭を確認します。

subimgs = copy.deepcopy(subimgs_all)
subctrs = copy.deepcopy(subctrs_all)

del subimgs[30]
del subctrs[30]

for i,(sims,lab,res,img,ctr) in enumerate(zip(svm_inputs, svm_labels, result[1], subimgs, subctrs)):
    if lab != res[0]:
        print('No.', i)
        print('{: }'.format(lab), ' -> ', '{: d}'.format(int(res[0])), ' [',end='')
        for s in sims: print('{:.3f}, '.format(s), end='');
        print(']')
        img = cv2.drawContours(img, [ctr], -1, (0,255,0), 1)
        plt.imshow(cv2.cvtColor(img, cv2.COLOR_BGR2RGB)),plt.xticks([]),plt.yticks([])
        plt.show()

    No. 16
    -1  ->   5  [0.796, 0.672, 0.715, 0.783, 0.878, ]

    No. 37
    -1  ->   0  [0.840, 0.879, 0.677, 0.662, 0.765, ]

    No. 52
    -1  ->   5  [0.780, 0.648, 0.735, 0.787, 0.825, ]

    No. 68
     0  ->  -1  [0.827, 0.785, 0.669, 0.627, 0.743, ]

    No. 76
    -1  ->   0  [0.942, 0.719, 0.693, 0.695, 0.736, ]

    No. 78
     0  ->  -1  [0.845, 0.789, 0.660, 0.674, 0.745, ]

    No. 94
     0  ->  -1  [0.843, 0.753, 0.688, 0.635, 0.697, ]

    No. 133
    -1  ->   0  [0.846, 0.798, 0.613, 0.641, 0.669, ]

    No. 136
    -1  ->   0  [0.867, 0.731, 0.655, 0.648, 0.766, ]

    No. 137
    -1  ->   0  [0.874, 0.761, 0.672, 0.641, 0.717, ]

    No. 139
    -1  ->   0  [0.908, 0.825, 0.694, 0.703, 0.659, ]

    No. 140
    -1  ->   0  [0.950, 0.822, 0.667, 0.693, 0.710, ]

    No. 141
    -1  ->   0  [0.894, 0.812, 0.611, 0.679, 0.680, ]

    No. 142
    -1  ->   0  [0.854, 0.730, 0.641, 0.644, 0.737, ]

    No. 143
    -1  ->   0  [0.894, 0.801, 0.696, 0.696, 0.714, ]

    No. 145
    -1  ->   0  [0.908, 0.816, 0.677, 0.696, 0.766, ]

    No. 147
    -1  ->   5  [0.804, 0.709, 0.707, 0.770, 0.848, ]

    No. 175
     0  ->  -1  [0.812, 0.814, 0.643, 0.620, 0.702, ]

    No. 188
    -1  ->   0  [0.909, 0.806, 0.682, 0.671, 0.690, ]

    No. 219
     0  ->  -1  [0.786, 0.820, 0.623, 0.615, 0.703, ]

    No. 242
    -1  ->   0  [0.960, 0.817, 0.667, 0.725, 0.724, ]

SVM再学習

判定に失敗した輪郭データの一部を学習データに加えます。
点数文字でない"5"が3つ見られますが、このうち2つを追加します。
その後再度判定を行ってみます。

train_data += [svm_inputs[16], svm_inputs[52]]
train_labels += [svm_labels[16], svm_labels[52]]
svm = prepare_svm(train_data, train_labels)

result = svm.predict(np.array(svm_inputs, 'float32'))
print_stat(result, svm_labels)

label -1 : {-1: 75, 0: 13, 1:  0, 2:  0, 3:  0, 5:  1, }
label  0 : {-1:  5, 0: 22, 1:  0, 2:  0, 3:  0, 5:  0, }
label  1 : {-1:  0, 0:  0, 1: 78, 2:  0, 3:  0, 5:  0, }
label  2 : {-1:  0, 0:  0, 1:  0, 2: 39, 3:  0, 5:  0, }
label  3 : {-1:  0, 0:  0, 1:  0, 2:  0, 3:  2, 5:  0, }
label  5 : {-1:  0, 0:  0, 1:  0, 2:  0, 3:  0, 5: 29, }

"5"の文字を全て正しく判別できるかと期待したが、そうはならず。
このまま進めてしまうこととします。

データ化

ここまでで得られたテンプレートデータとSVMモデルデータを保存し、点数計算アプリで使用したい。

SVMモデルの保存

SVMは保存するメソッドがあったので、それを使用します。
テンプレートデータについては、pythonでのオブジェクト保存の方法を調べるとpickle、jsonを使った2パターンが出てきます。 pickleでは、出所の分からないデータの読み込みを行うと意図しないコードを実行されてしまう、という脆弱性があるようですが、今回は自分で用意したデータを使いたいだけなので、一応問題はないかなと。
ただ、jsonを知っておいたほうが今後のためになりそうなので、jsonを使います。

Python公式 pickleドキュメント:

https://docs.python.org/ja/3/library/pickle.html

Python公式 jsonドキュメント:

https://docs.python.org/ja/3/library/json.html

Pythonでのjson参考:

https://hibiki-press.tech/python/json/1633
https://note.nkmk.me/python-json-load-dump/

svm.save('harupan_data/harupan_svm.dat')

生成されたデータの中身はこんな感じ。
テキストで、設定した内容も含めてSVMの情報が全部入っています。

%YAML:1.0
---
opencv_ml_svm:
   format: 3
   svmType: C_SVC
   kernel:
      type: LINEAR
   C: 100.
   term_criteria: { epsilon:1.1920928955078125e-07, iterations:1000 }
   var_count: 5
   class_count: 6
   class_labels: !!opencv-matrix
      rows: 6
      cols: 1
      dt: i
      data: [ -1, 0, 1, 2, 3, 5 ]
   sv_total: 15
   support_vectors:
      - [ -1.80685959e+01, -5.69025517e+00, 1.04596977e+01,
          7.75204945e+00, -2.17070246e+00 ]
      - [ 1.23291440e-01, -1.82865601e+01, -6.46431494e+00,
          4.22870070e-01, 6.76044846e+00 ]

...

      - [ -2.71152169e-01, -2.04700381e-01, 1.73537850e+00,
          3.85429978e+00, -5.29199266e+00 ]
   uncompressed_sv_total: 38
   uncompressed_support_vectors:
      - [ 8.49453330e-01, 7.48484850e-01, 6.99999988e-01, 6.75757587e-01,
          7.39393950e-01 ]
      - [ 8.43867898e-01, 7.64705896e-01, 6.90823317e-01, 7.08056450e-01,
          7.38181829e-01 ]
          
...
          
      - [ 7.79716969e-01, 6.48148119e-01, 7.35420227e-01, 7.86544859e-01,
          8.25454533e-01 ]
   decision_functions:
      -
         sv_count: 1
         rho: -9.4865848064106029e+00
         alpha: [ 1. ]
         index: [ 0 ]
      -
         sv_count: 1
         rho: -1.5486015742808821e+01
         alpha: [ 1. ]
         index: [ 1 ]
      -
         sv_count: 1
         rho: 5.0411393634237656e-01
         alpha: [ 1. ]
         index: [ 2 ]
         
...
         
      -
         sv_count: 1
         rho: 2.3617392745496142e+00
         alpha: [ 1. ]
         index: [ 13 ]
      -
         sv_count: 1
         rho: 1.9971712154702148e-01
         alpha: [ 1. ]
         index: [ 14 ]

保存データを読み込んで確認してみます。
Pythonでのモデル読み込みは以下を参照。

https://qiita.com/color_box/items/6f7d06fc3d65c6913ebf

svm_restored = cv2.ml.SVM_load('harupan_data/harupan_svm.dat')
result = svm_restored.predict(np.array(svm_inputs, 'float32'))
print_stat(result, svm_labels)

label -1 : {-1: 75, 0: 13, 1:  0, 2:  0, 3:  0, 5:  1, }
label  0 : {-1:  5, 0: 22, 1:  0, 2:  0, 3:  0, 5:  0, }
label  1 : {-1:  0, 0:  0, 1: 78, 2:  0, 3:  0, 5:  0, }
label  2 : {-1:  0, 0:  0, 1:  0, 2: 39, 3:  0, 5:  0, }
label  3 : {-1:  0, 0:  0, 1:  0, 2:  0, 3:  2, 5:  0, }
label  5 : {-1:  0, 0:  0, 1:  0, 2:  0, 3:  0, 5: 29, }

前と同じ結果が出ているので、問題なし。

テンプレートデータの保存(json)

テンプレートデータのjson保存のほうは、ndarrayがそのままでは扱えなかったので、リストに一回変換してから保存します。
読み込みのときに、ndarrayに戻します。

また、辞書データはjson形式と相性がいいようなので、これも組み合わせ。

https://wtnvenga.hatenablog.com/entry/2018/05/27/113848
https://note.nkmk.me/python-numpy-list/

テンプレートデータは、輪郭データと選択点リストを保存しておくこととします。
下準備の段階で、これらのデータを上に書いたクラスに与えてインスタンスを生成します。

import json

# ctr_list: List of contours for (0, 1, 2, 3, 5)
# pts_idx_list: List of selected point indices for (1, 2, 3, 5)
def save_templates(filename, ctr_list, pts_idx_list):
    with open(filename, mode='w') as f:
        save_data = []
        save_data += [{'num': 0, 'ctr': ctr_list[0].tolist(), 'pts': [0]}]
        for num, ctr, pts_idx in zip([1,2,3,5], ctr_list[1:5], pts_idx_list):
            save_data += [{'num': num, 'ctr': ctr.tolist(), 'pts': pts_idx}]
        json.dump(save_data, f, indent=2)
    return

def load_templates(filename):
    with open(filename, mode='r') as f:
        load_data = json.load(f)
        templates_rtn = []
        for d in load_data:
            templates_rtn += [template_dataset(np.array(d['ctr']), d['num'], d['pts'])]
    return templates_rtn

save_templates('harupan_data/templates2019.json', subctrs1, subctrs1_selected_pts)
save_templates('harupan_data/templates2020.json', subctrs3, subctrs3_selected_pts)
save_templates('harupan_data/templates2021.json', subctrs5, subctrs5_selected_pts)

生成されたデータは以下のような形。

[
  {
    "num": 0,
    "ctr": [
      [
        [
          18,
          52
        ]
      ],
      [
        [
          16,
          52
        ]
      ],
      [
        [
          15,
          52
        ]
      ],

...

      [
        [
          19,
          52
        ]
      ],
      [
        [
          19,
          52
        ]
      ]
    ],
    "pts": [
      0
    ]
  },
  {
    "num": 1,
    "ctr": [
      [
        [
          0,
          2
        ]
      ],
      [
        [
          1,
          0
        ]
      ],
      [
        [
          1,
          0
        ]
      ],

...

      [
        [
          0,
          4
        ]
      ],
      [
        [
          0,
          3
        ]
      ]
    ],
    "pts": [
      0,
      5,
      10,
      15,
      20,
      25,
      30,
      35,
      40,
      45,
      50,
      55,
      60,
      65,
      70,
      75,
      80,
      85,
      90,
      95,
      100,
      105,
      110,
      115,
      120,
      125,
      130,
      135,
      140
    ]
  },
  {
    "num": 2,
    "ctr": [
      [
        [
          12,
          1
        ]
      ],
      [
        [
          13,
          0
        ]
      ],

...

分かりやすくはないですが、直接見たいわけではないので特に問題なし。
テキスト保存なので無駄にデータ量が多くなりますが…。

きちんと復元できることも確認。

templates1_restored = load_templates('harupan_data/templates2019.json')
templates3_restored = load_templates('harupan_data/templates2020.json')
templates5_restored = load_templates('harupan_data/templates2021.json')

def disp_template(template):
    img = cv2.cvtColor(template.solid.copy(), cv2.COLOR_GRAY2RGB)
    if template.num != 0:
        img = cv2.drawContours(img, [template.ctr], -1, (0,255,0), 1)
        for p in template.pts:
            img = cv2.drawMarker(img, p, (255,0,0), markerType=cv2.MARKER_CROSS, markerSize=3)
    plt.imshow(img), plt.xticks([]), plt.yticks([])
    plt.show()

print('Template 2019')
for t in templates1_restored:
    disp_template(t)

print('Template 2020')
for t in templates3_restored:
    disp_template(t)

print('Template 2021')
for t in templates5_restored:
    disp_template(t)

    Template 2019

    Template 2020

    Template 2021

大丈夫そう。

スクリプト化

今までの処理を、pythonスクリプトとして保存、これを読み込めば必要な関数がロードされるようにしておきたいなと。

以下のスクリプト(harupan.py)を作成。
これでJupyter notebookがすっきりするか。

######################################################
# Importing libraries
######################################################
import cv2
import numpy as np
from matplotlib import pyplot as plt
import math
import copy
import random
import json

######################################################
# Detecting contours
######################################################
def detect_candidate_contours(image, res_th=800):
    h, w, chs = image.shape
    if h > res_th or w > res_th:
        k = float(res_th)/h if w > h else float(res_th)/w
    else:
        k = 1.0
    img = cv2.resize(image, None, fx=k, fy=k, interpolation=cv2.INTER_AREA)
    hsv = cv2.cvtColor(img, cv2.COLOR_BGR2HSV)
    # Convert hue value (rotation, mask by saturation)
    hsv[:,:,0] = np.where(hsv[:,:,0] < 50, hsv[:,:,0]+180, hsv[:,:,0])
    hsv[:,:,0] = np.where(hsv[:,:,1] < 100, 0, hsv[:,:,0])
    # Thresholding with cv2.inRange()
    th_hue = cv2.inRange(hsv[:,:,0], 135, 190)
    # Retrieve all points on the contours (cv2.CHAIN_APPROX_NONE)
    contours, hierarchy = cv2.findContours(th_hue, cv2.RETR_TREE, cv2.CHAIN_APPROX_NONE)
    indices0 = [i for i,hier in enumerate(hierarchy[0,:,:]) if hier[3] == -1]
    indices1 = [i for i,hier in enumerate(hierarchy[0,:,:]) if hier[3] in indices0]
    contours1 = [contours[i] for i in indices1]
    contours1_filtered = [ctr for ctr in contours1 if cv2.contourArea(ctr) > float(res_th)*float(res_th)/4000]
    return contours1_filtered, img


######################################################
# Auxiliary functions
######################################################
def create_contour_area_image(img, ctr):
    x,y,w,h = cv2.boundingRect(ctr)
    rtn_img = img[y:y+h,x:x+w,:].copy()
    rtn_ctr = ctr.copy()
    origin = np.array([x,y])
    for c in rtn_ctr:
        c[0,:] -= origin
    return rtn_img, rtn_ctr

# ctr: Should be output of create_contour_area_image() (Origin of points is the origin of bounding box)
# img_shape: Optional, tuple of (image_height, image_width), if omitted, calculated from ctr
def create_solid_contour(ctr, img_shape=(int(0),int(0))):
    if img_shape == (int(0),int(0)):
        _,_,w,h = cv2.boundingRect(ctr)
    else:
        h,w = img_shape
    img = np.zeros((h,w), 'uint8')
    img = cv2.drawContours(img, [ctr], -1, 255, -1)
    return img

# ctr: Should be output of create_contour_area_image() (Origin of points is the origin of bounding box)
def create_upright_solid_contour(ctr):
    (cx,cy),(w,h),angle = cv2.minAreaRect(ctr)
    M = cv2.getRotationMatrix2D((cx,cy), angle, 1)
    for i in range(ctr.shape[0]):
        ctr[i,0,:] = ( M @ np.array([ctr[i,0,0], ctr[i,0,1], 1]) ).astype('int')
    rect = cv2.boundingRect(ctr)
    img = np.zeros((rect[3],rect[2]), 'uint8')
    ctr -= rect[0:2]
    M[:,2] -= rect[0:2]
    img = cv2.drawContours(img, [ctr], -1, 255,-1)
    return img, M, ctr


######################################################
# Dataset classes
######################################################
class contour_dataset:
    def __init__(self, ctr):
        self.ctr = ctr.copy()
        self.rrect = cv2.minAreaRect(ctr)
        self.box = cv2.boxPoints(self.rrect)
        self.solid = create_solid_contour(ctr)
        self.pts = np.array([p for p in ctr[:,0,:]])

class template_dataset:
    def __init__(self, ctr, num, selected_idx=[0]):
        self.ctr = ctr.copy()
        self.num = num
        self.rrect = cv2.minAreaRect(ctr)
        self.box = cv2.boxPoints(self.rrect)
        if num == 0:
            self.solid,_,_ = create_upright_solid_contour(ctr)
        else:
            self.solid = create_solid_contour(ctr)
        self.pts = np.array([ctr[idx,0,:] for idx in selected_idx])


######################################################
# ICP
######################################################
# pts: list of 2D points, or ndarray of shape (n,2)
# query: 2D point to find nearest neighbor
def find_nearest_neighbor(pts, query):
    min_distance_sq = float('inf')
    min_idx = 0
    for i, p in enumerate(pts):
        d = np.dot(query - p, query - p)
        if(d < min_distance_sq):
            min_distance_sq = d
            min_idx = i
    return min_idx, np.sqrt(min_distance_sq)

# src, dst: ndarray, shape is (n,2) (n: number of points)
def estimate_affine_2d(src, dst):
    n = min(src.shape[0], dst.shape[0])
    x = dst[0:n].flatten()
    A = np.zeros((2*n,6))
    for i in range(n):
        A[i*2,0] = src[i,0]
        A[i*2,1] = src[i,1]
        A[i*2,2] = 1
        A[i*2+1,3] = src[i,0]
        A[i*2+1,4] = src[i,1]
        A[i*2+1,5] = 1
    M = np.linalg.inv(A.T @ A) @ A.T @ x
    return M.reshape([2,3])

# Find optimum affine matrix using ICP algorithm
# src_pts: ndarray, shape is (n_s,2) (n_s: number of points)
# dst_pts: ndarray, shape is (n_d,2) (n_d: number of points, n_d should be larger or equal to n_s)
# initial_matrix: ndarray, shape is (2,3)
def icp(src_pts, dst_pts, max_iter=20, initial_matrix=np.array([[1.0, 0.0, 0.0], [0.0, 1.0, 0.0]])):
    default_affine_matrix = np.array([[1.0, 0.0, 0.0], [0.0, 1.0, 0.0]])
    if dst_pts.shape[0] < src_pts.shape[0]:
        print("icp: Insufficient destination points")
        return default_affine_matrix, False
    if initial_matrix.shape != (2,3):
        print("icp: Illegal shape of initial_matrix")
        return default_affine_matrix, False
    M = initial_matrix
    # Store indices of the nearest neighbor point of dst_pts to the converted point of src_pts
    nn_idx = []
    for i in range(max_iter):
        nn_idx_tmp = []
        dst_pts_list = [p for p in dst_pts]
        idx_list = list(range(0,dst_pts.shape[0]))
        for p in src_pts:
            p2 = M @ np.array([p[0], p[1], 1])
            idx, d = find_nearest_neighbor(dst_pts_list, p2)
            nn_idx_tmp += [idx_list[idx]]
            del dst_pts_list[idx]
            del idx_list[idx]
        if nn_idx != [] and nn_idx == nn_idx_tmp:
            break
        dst_pts2 = np.zeros_like(src_pts)
        for j,idx in enumerate(nn_idx_tmp):
            dst_pts2[j,:] = dst_pts[idx,:]
        M = estimate_affine_2d(src_pts, dst_pts2)
        nn_idx = nn_idx_tmp
        if i == max_iter -1:
            return M, False
    return M, True


######################################################
# Calculating similarity and determining the number
######################################################
def binary_image_similarity(img1, img2):
    if img1.shape != img2.shape:
        print('binary_image_similarity: Different image size')
        return 0.0
    xor_img = cv2.bitwise_xor(img1, img2)
    return 1.0 - np.float(np.count_nonzero(xor_img)) / (img1.shape[0]*img2.shape[1])

# src, dst: contour_dataset or template_dataset (holding member variables box, solid)
def get_transform_by_rotated_rectangle(src, dst):
    # Rotated patterns are created when starting index is slided
    dst_box2 = np.vstack([dst.box, dst.box])
    max_similarity = 0.0
    max_converted_img = np.zeros((dst.solid.shape[1], dst.solid.shape[0]), 'uint8')
    for i in range(4):
        M = cv2.getAffineTransform(src.box[0:3], dst_box2[i:i+3])
        converted_img = cv2.warpAffine(src.solid, M, dsize=(dst.solid.shape[1], dst.solid.shape[0]), flags=cv2.INTER_NEAREST)
        similarity = binary_image_similarity(converted_img, dst.solid)
        if similarity > max_similarity:
            M_rtn = M
            max_similarity = similarity
            max_converted_img = converted_img
    return M_rtn, max_similarity, max_converted_img

def get_similarity_with_template(target_data, template_data, sim_th_high=0.95, sim_th_low=0.7):
    _,(w1,h1), _ = target_data.rrect
    _,(w2,h2), _ = template_data.rrect
    r = w1/h1 if w1 < h1 else h1/w1
    r = r * h2/w2 if w2 < h2 else r * w2/h2
    M, sim_init, _ = get_transform_by_rotated_rectangle(template_data, target_data)
    if sim_init > sim_th_high or sim_init < sim_th_low or r > 1.4 or r < 0.7:
        dsize = (template_data.solid.shape[1], template_data.solid.shape[0])
        flags = cv2.INTER_NEAREST|cv2.WARP_INVERSE_MAP
        converted_img = cv2.warpAffine(target_data.solid, M, dsize=dsize, flags=flags)
        return sim_init, converted_img
    M, _ = icp(template_data.pts, target_data.pts, initial_matrix=M)
    Minv = cv2.invertAffineTransform(M)
    converted_ctr = np.zeros_like(target_data.ctr)
    for i in range(target_data.ctr.shape[0]):
        converted_ctr[i,0,:] = (Minv[:,0:2] @ target_data.ctr[i,0,:]) + Minv[:,2]
    converted_img = create_solid_contour(converted_ctr, img_shape=template_data.solid.shape)
    val = binary_image_similarity(converted_img, template_data.solid)
    return val, converted_img

def get_similarity_with_template_zero(target_data, template_data):
    dsize = (template_data.solid.shape[1], template_data.solid.shape[0])
    converted_img = cv2.resize(target_data.solid, dsize=dsize, interpolation=cv2.INTER_NEAREST)
    val = binary_image_similarity(converted_img, template_data.solid)
    return val, converted_img

def get_similarities(target, templates):
    similarities = []
    converted_imgs = []
    for tmpl in templates:
        if tmpl.num == 0:
            sim,converted_img = get_similarity_with_template_zero(target, tmpl)
        else:
            sim,converted_img = get_similarity_with_template(target, tmpl)
        similarities += [sim]
        converted_imgs += [converted_img]
    return similarities, converted_imgs

# target: Single contour to compare
# templates: List of template_dataset (for numbers 0, 1, 2, 3, 5)
# svm: Trained SVM
# return: determined number (0,1,2,3,5), -1 if none corresponds
def determine_number(target, templates, svm):
    similarities,_ = get_similarities(target, templates)
    _, result = svm.predict(np.array(similarities))
    return int(result[0])


######################################################
# Loading template data and SVM model
######################################################
def load_svm(filename):
    return cv2.ml.SVM_load(filename)

def load_templates(filename):
    with open(filename, mode='r') as f:
        load_data = json.load(f)
        templates_rtn = []
        for d in load_data:
            templates_rtn += [template_dataset(np.array(d['ctr']), d['num'], d['pts'])]
    return templates_rtn

ここまで

春のパン祭りシール点数集計ではまだやらないといけないことがあって、

• 点数計算の処理
• 1枚の入力画像を受けてから点数を計算するまでの一連の流れ

というところですが、また次回にします。