On Detecting and Removing Superficial Redundancy in Vector Databases

<table class="fixed-width table-group" id="tab7"><tr><td><table class="table"><colgroup><col style="width:6.34em"/><col style="width:4.21em"/><col style="width:6.06em"/></colgroup><tr><td class="thead-hr" colspan="3"><hr/></td></tr><tr class="thead"><td class="align_left">Redundancy </td><td class="align_center"> Python </td><td class="align_center"> MapReduce</td></tr><tr><td class="thead-hr" colspan="3"><hr/></td></tr><tr><td class="align_left"> <svg height="12.3916pt" id="M214" style="vertical-align:-3.42948pt" version="1.1" viewbox="-0.0498162 -8.96212 32.4735 12.3916" width="32.4735pt" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink"><g transform="matrix(.013,0,0,-0.013,0,0)"><path d="M620 675H597C578 656 570 650 541 650H144C112 650 104 653 94 675H72C59 618 42 552 23 493L53 491C71 534 88 564 105 585C124 608 144 615 238 615H290L197 121C182 40 174 34 88 28L82 0H361L367 28C275 34 266 38 281 121L374 615H441C522 615 543 608 553 583C562 560 566 531 565 493L597 494C603 551 612 629 620 675Z" id="g113-85"></path></g><g transform="matrix(.013,0,0,-0.013,6.98,0)"><path d="M556 393C556 426 537 448 514 448C496 448 478 435 471 420C468 414 462 401 466 394C473 382 480 368 480 346C480 268 392 143 338 67H336C329 163 319 253 309 330S286 448 254 448C215 448 160 383 127 337L143 311C167 344 200 373 208 373S222 365 229 320C246 214 264 66 268 -20C219 -83 131 -171 17 -239L25 -261L137 -235C248 -110 273 -79 335 8C384 77 481 215 520 287C544 332 556 363 556 393Z" id="g113-122"></path></g><g transform="matrix(.013,0,0,-0.013,13.596,0)"><path d="M570 304C570 398 525 448 414 448C385 448 343 445 312 434L329 511L321 518C297 504 262 482 244 460L233 411C195 397 159 381 128 358L135 332C160 347 189 360 224 373L111 -147C97 -210 84 -218 17 -231L13 -257L254 -247L259 -218L233 -216C183 -212 177 -202 189 -142L218 -1C238 -10 266 -12 283 -12C351 3 429 48 483 105C543 168 570 242 570 304ZM482 289C482 161 380 33 304 33C278 33 248 51 233 69L303 396C326 400 352 403 369 403C428 403 482 380 482 289Z" id="g113-113"></path></g><g transform="matrix(.013,0,0,-0.013,21.306,0)"><path d="M391 364C391 409 353 448 295 448C249 448 198 426 152 393C65 331 23 225 23 139C23 14 96 -12 146 -12C198 -12 280 9 367 101L351 124C300 78 242 48 194 48C129 48 109 107 109 162V191C208 213 391 266 391 364ZM313 350C313 305 268 261 113 223C132 334 187 381 217 398C227 404 244 405 261 405C290 405 313 385 313 350Z" id="g113-102"></path></g><g transform="matrix(.013,0,0,-0.013,26.688,0)"><path d="M405 650H141L135 622C222 616 230 610 215 535L133 116C118 41 113 33 29 28L23 0H289L295 28C209 33 205 40 219 116L298 535C312 609 317 616 399 622L405 650Z" id="g113-74"></path></g></svg></td><td class="align_center"> 13 sec</td><td class="align_center"> 6 min 10 sec</td></tr><tr><td class="align_left"> <svg height="12.3916pt" id="M215" style="vertical-align:-3.42948pt" version="1.1" viewbox="-0.0498162 -8.96212 37.4059 12.3916" width="37.4059pt" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink"><g transform="matrix(.013,0,0,-0.013,0,0)"><path d="M620 675H597C578 656 570 650 541 650H144C112 650 104 653 94 675H72C59 618 42 552 23 493L53 491C71 534 88 564 105 585C124 608 144 615 238 615H290L197 121C182 40 174 34 88 28L82 0H361L367 28C275 34 266 38 281 121L374 615H441C522 615 543 608 553 583C562 560 566 531 565 493L597 494C603 551 612 629 620 675Z" id="g113-85"></path></g><g transform="matrix(.013,0,0,-0.013,6.98,0)"><path d="M556 393C556 426 537 448 514 448C496 448 478 435 471 420C468 414 462 401 466 394C473 382 480 368 480 346C480 268 392 143 338 67H336C329 163 319 253 309 330S286 448 254 448C215 448 160 383 127 337L143 311C167 344 200 373 208 373S222 365 229 320C246 214 264 66 268 -20C219 -83 131 -171 17 -239L25 -261L137 -235C248 -110 273 -79 335 8C384 77 481 215 520 287C544 332 556 363 556 393Z" id="g113-122"></path></g><g transform="matrix(.013,0,0,-0.013,13.596,0)"><path d="M570 304C570 398 525 448 414 448C385 448 343 445 312 434L329 511L321 518C297 504 262 482 244 460L233 411C195 397 159 381 128 358L135 332C160 347 189 360 224 373L111 -147C97 -210 84 -218 17 -231L13 -257L254 -247L259 -218L233 -216C183 -212 177 -202 189 -142L218 -1C238 -10 266 -12 283 -12C351 3 429 48 483 105C543 168 570 242 570 304ZM482 289C482 161 380 33 304 33C278 33 248 51 233 69L303 396C326 400 352 403 369 403C428 403 482 380 482 289Z" id="g113-113"></path></g><g transform="matrix(.013,0,0,-0.013,21.306,0)"><path d="M391 364C391 409 353 448 295 448C249 448 198 426 152 393C65 331 23 225 23 139C23 14 96 -12 146 -12C198 -12 280 9 367 101L351 124C300 78 242 48 194 48C129 48 109 107 109 162V191C208 213 391 266 391 364ZM313 350C313 305 268 261 113 223C132 334 187 381 217 398C227 404 244 405 261 405C290 405 313 385 313 350Z" id="g113-102"></path></g><g transform="matrix(.013,0,0,-0.013,26.688,0)"><path d="M405 650H141L135 622C222 616 230 610 215 535L133 116C118 41 113 33 29 28L23 0H289L295 28C209 33 205 40 219 116L298 535C312 609 317 616 399 622L405 650Z" id="g113-74"></path></g><g transform="matrix(.013,0,0,-0.013,31.603,0)"><path d="M405 650H141L135 622C222 616 230 610 215 535L133 116C118 41 113 33 29 28L23 0H289L295 28C209 33 205 40 219 116L298 535C312 609 317 616 399 622L405 650Z" id="g113-74"></path></g></svg></td><td class="align_center"> 20 sec</td><td class="align_center"> 6 min 3 sec</td></tr><tr><td class="align_left"> <svg height="12.3916pt" id="M216" style="vertical-align:-3.42948pt" version="1.1" viewbox="-0.0498162 -8.96212 42.3383 12.3916" width="42.3383pt" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink"><g transform="matrix(.013,0,0,-0.013,0,0)"><path d="M620 675H597C578 656 570 650 541 650H144C112 650 104 653 94 675H72C59 618 42 552 23 493L53 491C71 534 88 564 105 585C124 608 144 615 238 615H290L197 121C182 40 174 34 88 28L82 0H361L367 28C275 34 266 38 281 121L374 615H441C522 615 543 608 553 583C562 560 566 531 565 493L597 494C603 551 612 629 620 675Z" id="g113-85"></path></g><g transform="matrix(.013,0,0,-0.013,6.98,0)"><path d="M556 393C556 426 537 448 514 448C496 448 478 435 471 420C468 414 462 401 466 394C473 382 480 368 480 346C480 268 392 143 338 67H336C329 163 319 253 309 330S286 448 254 448C215 448 160 383 127 337L143 311C167 344 200 373 208 373S222 365 229 320C246 214 264 66 268 -20C219 -83 131 -171 17 -239L25 -261L137 -235C248 -110 273 -79 335 8C384 77 481 215 520 287C544 332 556 363 556 393Z" id="g113-122"></path></g><g transform="matrix(.013,0,0,-0.013,13.596,0)"><path d="M570 304C570 398 525 448 414 448C385 448 343 445 312 434L329 511L321 518C297 504 262 482 244 460L233 411C195 397 159 381 128 358L135 332C160 347 189 360 224 373L111 -147C97 -210 84 -218 17 -231L13 -257L254 -247L259 -218L233 -216C183 -212 177 -202 189 -142L218 -1C238 -10 266 -12 283 -12C351 3 429 48 483 105C543 168 570 242 570 304ZM482 289C482 161 380 33 304 33C278 33 248 51 233 69L303 396C326 400 352 403 369 403C428 403 482 380 482 289Z" id="g113-113"></path></g><g transform="matrix(.013,0,0,-0.013,21.306,0)"><path d="M391 364C391 409 353 448 295 448C249 448 198 426 152 393C65 331 23 225 23 139C23 14 96 -12 146 -12C198 -12 280 9 367 101L351 124C300 78 242 48 194 48C129 48 109 107 109 162V191C208 213 391 266 391 364ZM313 350C313 305 268 261 113 223C132 334 187 381 217 398C227 404 244 405 261 405C290 405 313 385 313 350Z" id="g113-102"></path></g><g transform="matrix(.013,0,0,-0.013,26.688,0)"><path d="M405 650H141L135 622C222 616 230 610 215 535L133 116C118 41 113 33 29 28L23 0H289L295 28C209 33 205 40 219 116L298 535C312 609 317 616 399 622L405 650Z" id="g113-74"></path></g><g transform="matrix(.013,0,0,-0.013,31.603,0)"><path d="M405 650H141L135 622C222 616 230 610 215 535L133 116C118 41 113 33 29 28L23 0H289L295 28C209 33 205 40 219 116L298 535C312 609 317 616 399 622L405 650Z" id="g113-74"></path></g><g transform="matrix(.013,0,0,-0.013,36.518,0)"><path d="M405 650H141L135 622C222 616 230 610 215 535L133 116C118 41 113 33 29 28L23 0H289L295 28C209 33 205 40 219 116L298 535C312 609 317 616 399 622L405 650Z" id="g113-74"></path></g></svg></td><td class="align_center"> 6 sec</td><td class="align_center"> 6 min 21 sec</td></tr><tr class="table-tr"><td colspan="3"><hr class="tbody-hr"/></td></tr></table></td></tr></table>

<div>Time needed to complete the tasks in Python and MapReduce.</div>

Mathematical Problems in Engineering

tab7

Table 7

Table 7: On Detecting and Removing Superficial Redundancy in Vector Databases 