| (i) | Input: recorded video RV |
| (ii) | Output: test sequence script TSS |
| (1) | split_tag = 0 |
| (2) | for each frame in RV do |
| (3) | finger = getOpenPoseDetection(RV) |
| (4) | if finger is exist then |
| (5) | gesture_info ⇐ map(frame, split_tag, finger) |
| (6) | else |
| (7) | gui_info ⇐ map(frame, split_tag) |
| (8) | if pre_frames is gesture frame then |
| (9) | split_tag ++ |
| (10) | end if |
| (11) | end if |
| (12) | end for |
| (13) | for st = 0 to split_tag do |
| (14) | gui = getMiddleFrame(gui_info, st) |
| (15) | gui_elements = getObjectDetection(gui) |
| (16) | gui_skeleton = getGuiSkeleton(gui_elements) |
| (17) | gesture_micro = identifyGesture(gesture_info, st) |
| (18) | object = identifyObject(gesture_micro, gui_elements) |
| (19) | TTS ⇐ getTestSquence(st, gui_skeleton, gesture_micro, object) |
| (20) | end for |
| (21) | return TSS |
| (22) | function identifyGesture(gesture_info, st) |
| (23) | t = getInterMicroFrames(gesture_info,st,f_threshold) |
| (24) | if |coord_firstframe(t) − coord_endframe(t)| < c_threshold then |
| (25) | return map(move, coord_first, coord_end) |
| (26) | else if length(t) < n_threshold then |
| (27) | return map(none, coord_first, null) |
| (28) | else if length(t) > h_threshold then |
| (29) | return map(hold, coord_first, null) |
| (30) | else |
| (31) | return map(tap, coord_first, null) |
| (32) | end if |
| (33) | end function |
| (34) | function identifyObject(gesture_micro, gui_elements) |
| (35) | for each element in gui_elements do |
| (36) | isContain = contain(coord_first, element) |
| (37) | if isContain = true then |
| (38) | return element |
| (39) | end if |
| (40) | end for |
| (41) | return surface |
| (42) | end function |
