Multimodal Semantics Extraction from User-Generated Videos

<table class="table-group" id="tab4"><tr><td><table class="table"><tr><td class="thead-hr" colspan="7"><hr/></td></tr><tr><td align="left"></td><td align="center"></td><td align="center" colspan="5">Automatic event genre classification</td></tr><tr><td align="left">Event</td><td align="center">Ground truth event genre</td><td align="center">Feature-set <svg height="15.6875" id="M65" style="vertical-align:-3.39066pt" version="1.1" viewbox="0 0 14.6125 15.6875" width="14.6125" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
<g transform="matrix(.017,0,0,-.017,.062,11.4)"><path d="M457 488l-30 -3q-17 148 -131 148q-53 0 -84.5 -34.5t-31.5 -82.5q0 -42 25.5 -72t74.5 -62l33 -22q63 -42 95 -85t32 -102q0 -84 -67 -137t-163 -53q-58 0 -113 22t-70 43l-4 152l27 4q4 -32 15 -62.5t31 -59.5t53.5 -47t76.5 -18q56 0 92 35t36 96q0 39 -25 70t-78 68
l-31 22q-32 23 -53.5 41.5t-45 57t-23.5 77.5q0 82 58 132.5t156 50.5q46 0 101 -17l18.5 -6t17 -6t8.5 -3q-4 -55 0 -147z" id="x1D446"></path></g>
<g transform="matrix(.012,0,0,-.012,8.225,15.475)"><path d="M153 550l-26 -186q79 31 111 31q90 0 141.5 -51t51.5 -119q0 -93 -89 -166q-85 -69 -173 -71q-32 0 -61.5 11.5t-41.5 23.5q-18 17 -17 34q2 16 22 33q14 9 26 -1q61 -50 124 -50q60 0 93 43.5t33 104.5q0 69 -41.5 110t-121.5 41q-53 0 -102 -20l38 305h286l6 -8
l-26 -65h-233z" id="x35"></path></g>
</svg> (audio, sensors)</td><td align="center">Feature-set <svg height="15.6875" id="M66" style="vertical-align:-3.39066pt" version="1.1" viewbox="0 0 14.6125 15.6875" width="14.6125" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
<g transform="matrix(.017,0,0,-.017,.062,11.4)"><path d="M457 488l-30 -3q-17 148 -131 148q-53 0 -84.5 -34.5t-31.5 -82.5q0 -42 25.5 -72t74.5 -62l33 -22q63 -42 95 -85t32 -102q0 -84 -67 -137t-163 -53q-58 0 -113 22t-70 43l-4 152l27 4q4 -32 15 -62.5t31 -59.5t53.5 -47t76.5 -18q56 0 92 35t36 96q0 39 -25 70t-78 68
l-31 22q-32 23 -53.5 41.5t-45 57t-23.5 77.5q0 82 58 132.5t156 50.5q46 0 101 -17l18.5 -6t17 -6t8.5 -3q-4 -55 0 -147z" id="x1D446"></path></g>
<g transform="matrix(.012,0,0,-.012,8.225,15.475)"><path d="M137 343l67 33q37 17 63 17q79 0 129.5 -53t50.5 -131q0 -92 -58 -156.5t-147 -64.5t-147 68t-58 182q0 63 17 119t43 95.5t61.5 72t69 52t67.5 31.5q62 22 128 33l6 -32q-56 -11 -108 -35q-149 -71 -184 -231zM227 337q-47 0 -95 -27q-6 -23 -6 -70q0 -93 36 -155.5
t96 -62.5q53 0 78 45.5t25 105.5q0 68 -35 116t-99 48z" id="x36"></path></g>
</svg> (DSIFT, sensors)</td><td align="center">Feature-set <svg height="15.6875" id="M67" style="vertical-align:-3.39066pt" version="1.1" viewbox="0 0 14.6125 15.6875" width="14.6125" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
<g transform="matrix(.017,0,0,-.017,.062,11.4)"><path d="M457 488l-30 -3q-17 148 -131 148q-53 0 -84.5 -34.5t-31.5 -82.5q0 -42 25.5 -72t74.5 -62l33 -22q63 -42 95 -85t32 -102q0 -84 -67 -137t-163 -53q-58 0 -113 22t-70 43l-4 152l27 4q4 -32 15 -62.5t31 -59.5t53.5 -47t76.5 -18q56 0 92 35t36 96q0 39 -25 70t-78 68
l-31 22q-32 23 -53.5 41.5t-45 57t-23.5 77.5q0 82 58 132.5t156 50.5q46 0 101 -17l18.5 -6t17 -6t8.5 -3q-4 -55 0 -147z" id="x1D446"></path></g>
<g transform="matrix(.012,0,0,-.012,8.225,15.475)"><path d="M447 623l8 -12l-283 -613l-74 -10l-7 11q174 283 297 551h-216q-48 0 -62.5 -12t-33.5 -63h-29q10 60 18 148h382z" id="x37"></path></g>
</svg> (global visual, sensors)</td><td align="center">Feature-set <svg height="15.6875" id="M68" style="vertical-align:-3.39066pt" version="1.1" viewbox="0 0 14.6125 15.6875" width="14.6125" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
<g transform="matrix(.017,0,0,-.017,.062,11.4)"><path d="M457 488l-30 -3q-17 148 -131 148q-53 0 -84.5 -34.5t-31.5 -82.5q0 -42 25.5 -72t74.5 -62l33 -22q63 -42 95 -85t32 -102q0 -84 -67 -137t-163 -53q-58 0 -113 22t-70 43l-4 152l27 4q4 -32 15 -62.5t31 -59.5t53.5 -47t76.5 -18q56 0 92 35t36 96q0 39 -25 70t-78 68
l-31 22q-32 23 -53.5 41.5t-45 57t-23.5 77.5q0 82 58 132.5t156 50.5q46 0 101 -17l18.5 -6t17 -6t8.5 -3q-4 -55 0 -147z" id="x1D446"></path></g>
<g transform="matrix(.012,0,0,-.012,8.225,15.475)"><path d="M249 635q70 0 116 -43t46 -105q0 -46 -28 -80q-22 -25 -80 -64q62 -35 97 -75t35 -99q0 -81 -63 -131t-135 -50q-83 0 -137.5 45.5t-54.5 123.5q0 52 45 95q29 28 89 64q-109 62 -109 155q0 66 50.5 115t128.5 49zM238 603q-42 0 -67.5 -31t-25.5 -72q0 -50 32.5 -79.5
t98.5 -62.5q61 48 61 124q0 59 -29.5 90t-69.5 31zM248 20q46 0 76.5 33.5t30.5 89.5q0 50 -39 85.5t-110 71.5q-81 -54 -81 -137q0 -67 35.5 -105t87.5 -38z" id="x38"></path></g>
</svg> (audio, DSIFT, sensors)</td><td align="center">Feature-set <svg height="15.725" id="M69" style="vertical-align:-3.41933pt" version="1.1" viewbox="0 0 14.6125 15.725" width="14.6125" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
<g transform="matrix(.017,0,0,-.017,.062,11.4)"><path d="M457 488l-30 -3q-17 148 -131 148q-53 0 -84.5 -34.5t-31.5 -82.5q0 -42 25.5 -72t74.5 -62l33 -22q63 -42 95 -85t32 -102q0 -84 -67 -137t-163 -53q-58 0 -113 22t-70 43l-4 152l27 4q4 -32 15 -62.5t31 -59.5t53.5 -47t76.5 -18q56 0 92 35t36 96q0 39 -25 70t-78 68
l-31 22q-32 23 -53.5 41.5t-45 57t-23.5 77.5q0 82 58 132.5t156 50.5q46 0 101 -17l18.5 -6t17 -6t8.5 -3q-4 -55 0 -147z" id="x1D446"></path></g>
<g transform="matrix(.012,0,0,-.012,8.225,15.475)"><path d="M244 635q90 0 143 -72t53 -177q0 -133 -65 -229.5t-171 -139.5q-79 -32 -140 -32l-5 30q109 18 185 91t101 186l-68 -36q-29 -16 -60 -16q-79 0 -129 51.5t-50 130.5q0 80 57 146.5t149 66.5zM228 602q-52 0 -78 -45.5t-26 -98.5q0 -69 36.5 -115.5t97.5 -46.5
q53 0 90 28q4 31 4 66q0 51 -9.5 95.5t-39 80.5t-75.5 36z" id="x39"></path></g>
</svg> (audio, global visual, sensors)—Proposed set</td></tr><tr><td align="left" colspan="7"><hr/></td></tr><tr><td align="left">Football match 1</td><td align="center">Sport</td><td align="center">Sport</td><td align="center">Sport</td><td align="center">Sport</td><td align="center">Sport</td><td align="center">Sport</td></tr><tr><td align="left">Football match 2</td><td align="center">Sport</td><td align="center">Sport</td><td align="center">Sport</td><td align="center">Sport</td><td align="center">Sport</td><td align="center">Sport</td></tr><tr><td align="left">Football match 3</td><td align="center">Sport</td><td align="center">Sport</td><td align="center">Sport</td><td align="center">Sport</td><td align="center">Sport</td><td align="center">Sport</td></tr><tr><td align="left">Ice-hockey match 1</td><td align="center">Sport</td><td align="center">Sport</td><td align="center">Sport</td><td align="center">Sport</td><td align="center">Live music</td><td align="center">Sport</td></tr><tr><td align="left">Ice-hockey match 2</td><td align="center">Sport</td><td align="center">Sport</td><td align="center">Sport</td><td align="center">Sport</td><td align="center">Live music</td><td align="center">Sport</td></tr><tr><td align="left">Concert 1</td><td align="center">Live music</td><td align="center">Sport</td><td align="center">Sport</td><td align="center">Sport</td><td align="center">Live music</td><td align="center">Live music</td></tr><tr><td align="left">Concert 2</td><td align="center">Live music</td><td align="center">Live music</td><td align="center">Live music</td><td align="center">Live music</td><td align="center">Live music</td><td align="center">Live music</td></tr><tr><td align="left">Concert 3</td><td align="center">Live music</td><td align="center">Live music</td><td align="center">Live music</td><td align="center">Live music</td><td align="center">Live music</td><td align="center">Live music</td></tr><tr><td align="left">Concert 4</td><td align="center">Live music</td><td align="center">Live music</td><td align="center">Live music</td><td align="center">Live music</td><td align="center">Live music</td><td align="center">Live music</td></tr><tr><td align="left" colspan="7"><hr/></td></tr><tr><td align="left">Total accuracy (%)</td><td align="center">—</td><td align="center">88.9</td><td align="center">88.9</td><td align="center">88.9</td><td align="center">77.8</td><td align="center">100</td></tr><tr class="table-tr"><td colspan="7"><hr class="tbody-hr"/></td></tr></table></td></tr></table>

Performance comparison for the event genre classification task using different feature-sets.

Advances in Multimedia

Multimodal Semantics Extraction from User-Generated Videos

Table 4