Detection Method for Distributed Web-Crawlers: A Long-Tail Threshold Model

<table class="fixed-width table-group" id="tab1"><tr><td><table class="table"><colgroup><col style="width:6.19em"/><col style="width:5.46em"/><col style="width:5.22em"/><col style="width:5.08em"/></colgroup><tr><td class="thead-hr" colspan="4"><hr/></td></tr><tr class="thead"><td class="align_left"> </td><td class="align_center"># of access</td><td class="align_center"># of Items</td><td class="align_center"># of users</td></tr><tr><td class="thead-hr" colspan="4"><hr/></td></tr><tr><td class="align_left"> <svg height="8.8423pt" id="M29" style="vertical-align:-0.2064009pt" version="1.1" viewbox="-0.0498162 -8.6359 34.1177 8.8423" width="34.1177pt" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink"><g transform="matrix(.013,0,0,-0.013,0,0)"><path d="M822 650H589L583 622C660 617 677 607 674 561C672 534 664 481 647 390L600 137H596L273 650H126L120 622C176 620 194 615 207 594C221 571 225 557 214 504L161 257C141 166 129 112 121 85C108 42 83 30 29 28L23 0H260L266 28C193 33 173 42 176 89C178 122 186 172 202 255L256 527H259L583 -8H612L690 390C708 481 720 535 728 558C744 603 756 619 816 622L822 650Z"></path></g><g transform="matrix(.013,0,0,-0.013,9.997,0)"><path d="M686 28C612 35 607 44 591 112C563 234 541 360 519 489L489 666L457 658L147 121C100 40 89 36 24 28L17 0H240L250 28C168 34 159 41 190 101L262 237H482C495 180 503 137 510 91C517 47 514 35 441 28L433 0H677L686 28ZM475 280H285L429 541H431L475 280Z"></path></g><g transform="matrix(.013,0,0,-0.013,19.132,0)"><path d="M449 634C442 637 425 643 405 650C376 660 341 666 307 666C181 666 98 590 98 485C98 400 170 343 215 310L246 288C307 243 343 204 343 147C343 67 291 18 219 18C104 18 61 124 51 202L23 199C28 124 27 71 27 47C47 22 122 -16 204 -16C324 -16 428 60 428 174C428 256 379 309 307 360L276 382C223 419 179 455 179 516C179 576 221 632 293 632C379 632 410 564 418 487L448 490C446 536 446 592 449 634Z"></path></g><g transform="matrix(.013,0,0,-0.013,24.746,0)"><path d="M686 28C612 35 607 44 591 112C563 234 541 360 519 489L489 666L457 658L147 121C100 40 89 36 24 28L17 0H240L250 28C168 34 159 41 190 101L262 237H482C495 180 503 137 510 91C517 47 514 35 441 28L433 0H677L686 28ZM475 280H285L429 541H431L475 280Z"></path></g></svg></td><td class="align_center">1,701,011</td><td class="align_center">7,649</td><td class="align_center">76,040</td></tr><tr><td class="align_left"> <svg height="12.7178pt" id="M30" style="vertical-align:-3.42947pt" version="1.1" viewbox="-0.0498162 -9.28833 61.768 12.7178" width="61.768pt" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink"><g transform="matrix(.013,0,0,-0.013,0,0)"><path d="M828 650H570L564 622C646 614 650 609 634 524L604 366H253L281 524C296 608 308 615 382 622L388 650H132L126 622C211 615 214 609 198 524L121 122C106 42 102 35 23 28L17 0H276L282 28C196 35 191 40 206 122L243 324H597L559 123C544 42 537 35 452 28L446 0H710L716 28C632 35 628 43 643 122L719 524C735 610 741 615 822 622L828 650Z"></path></g><g transform="matrix(.013,0,0,-0.013,9.488,0)"><path d="M556 393C556 426 537 448 514 448C496 448 478 435 471 420C468 414 462 401 466 394C473 382 480 368 480 346C480 268 392 143 338 67H336C329 163 319 253 309 330S286 448 254 448C215 448 160 383 127 337L143 311C167 344 200 373 208 373S222 365 229 320C246 214 264 66 268 -20C219 -83 131 -171 17 -239L25 -261L137 -235C248 -110 273 -79 335 8C384 77 481 215 520 287C544 332 556 363 556 393Z"></path></g><g transform="matrix(.013,0,0,-0.013,16.886,0)"><path d="M530 686C535 705 530 712 521 712C504 712 448 684 359 674L358 648H393C437 648 439 646 429 593L400 435C372 447 345 448 332 448C286 448 194 414 144 373C68 311 23 203 23 111C23 26 57 -12 91 -12C120 -12 147 3 188 29C227 54 290 102 341 170H343L322 71C308 6 320 -12 341 -12C373 -12 442 27 501 96L485 120C455 91 422 67 408 67C401 67 401 76 404 91C440 294 479 473 530 686ZM387 375L355 241C326 187 200 53 142 53C126 53 109 73 109 130C109 217 154 337 218 381C240 396 265 404 297 404S372 390 387 375Z"></path></g><g transform="matrix(.013,0,0,-0.013,24.102,0)"><path d="M391 364C391 409 353 448 295 448C249 448 198 426 152 393C65 331 23 225 23 139C23 14 96 -12 146 -12C198 -12 280 9 367 101L351 124C300 78 242 48 194 48C129 48 109 107 109 162V191C208 213 391 266 391 364ZM313 350C313 305 268 261 113 223C132 334 187 381 217 398C227 404 244 405 261 405C290 405 313 385 313 350Z"></path></g><g transform="matrix(.013,0,0,-0.013,29.484,0)"><path d="M393 379C402 394 400 411 393 422C384 437 365 448 348 448C301 448 237 372 186 285H182L193 335C210 408 205 448 178 448C150 448 80 402 29 344L45 321C80 355 114 373 122 373C128 373 130 365 124 330C106 228 76 98 50 -5L57 -12C82 -5 112 3 132 6L172 203C196 256 234 304 254 329C275 355 293 367 306 367C318 367 330 360 342 348C347 343 355 343 365 350S386 367 393 379Z"></path></g><g transform="matrix(.013,0,0,-0.013,34.972,0)"><path d="M483 97L471 123C436 91 401 65 392 65C388 65 384 74 390 106C414 239 444 378 457 429L455 433C444 433 429 436 416 439C392 444 368 448 344 448C281 448 204 415 152 376C71 315 23 205 23 103C23 21 57 -12 85 -12C114 -12 149 6 185 34C231 70 285 119 329 183H331L309 81C292 0 308 -12 326 -12C350 -12 421 24 483 97ZM374 387C370 363 356 291 345 261C315 193 181 50 139 50C124 50 110 71 110 118C110 224 153 331 218 379C238 394 271 402 301 402C329 402 359 394 374 387Z"></path></g><g transform="matrix(.013,0,0,-0.013,41.552,0)"><path d="M452 333C452 398 425 448 375 448C329 448 235 404 140 292H138L229 683C233 701 234 712 225 712C208 712 149 686 66 677L64 652H97C135 652 140 651 129 598L38 167C23 96 29 57 44 33C60 7 98 -12 132 -12C169 -12 232 3 290 41C383 102 452 223 452 333ZM365 316C365 199 308 87 239 49C221 39 200 35 183 35C137 35 99 70 112 163C116 191 123 215 129 233C190 316 290 392 330 392C348 392 365 372 365 316Z"></path></g><g transform="matrix(.013,0,0,-0.013,47.654,0)"><path d="M483 97L471 123C436 91 401 65 392 65C388 65 384 74 390 106C414 239 444 378 457 429L455 433C444 433 429 436 416 439C392 444 368 448 344 448C281 448 204 415 152 376C71 315 23 205 23 103C23 21 57 -12 85 -12C114 -12 149 6 185 34C231 70 285 119 329 183H331L309 81C292 0 308 -12 326 -12C350 -12 421 24 483 97ZM374 387C370 363 356 291 345 261C315 193 181 50 139 50C124 50 110 71 110 118C110 224 153 331 218 379C238 394 271 402 301 402C329 402 359 394 374 387Z"></path></g><g transform="matrix(.013,0,0,-0.013,54.234,0)"><path d="M530 686C535 705 530 712 521 712C504 712 448 684 359 674L358 648H393C437 648 439 646 429 593L400 435C372 447 345 448 332 448C286 448 194 414 144 373C68 311 23 203 23 111C23 26 57 -12 91 -12C120 -12 147 3 188 29C227 54 290 102 341 170H343L322 71C308 6 320 -12 341 -12C373 -12 442 27 501 96L485 120C455 91 422 67 408 67C401 67 401 76 404 91C440 294 479 473 530 686ZM387 375L355 241C326 187 200 53 142 53C126 53 109 73 109 130C109 217 154 337 218 381C240 396 265 404 297 404S372 390 387 375Z"></path></g></svg></td><td class="align_center">3,208,200</td><td class="align_center">16</td><td class="align_center">81,087</td></tr><tr class="table-tr"><td colspan="4"><hr class="tbody-hr"/></td></tr></table></td></tr></table>

Security and Communication Networks

tab1

Table 1

Table 1: Detection Method for Distributed Web-Crawlers: A Long-Tail Threshold Model 