Horseshoe Chaos in a 3D Neural Network with Different Activation Functions

<table>The image <svg height="16.275" id="M207" style="vertical-align:-3.27605pt" version="1.1" viewbox="0 0 104.8 16.275" width="104.8" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
<g transform="matrix(.017,-0,0,-.017,.062,12.137)"><path d="M574 449q-23 -58 -38 -79q-16 -4 -79 -4q-27 -100 -46 -219q-14 -87 7 -87t74 36l13 -27q-74 -81 -139 -81q-48 0 -48 62q0 22 8 59l60 258l-154 4q-21 -103 -54.5 -209t-64.5 -151q-38 -23 -83 -23l-7 15q46 36 94 152.5t64 216.5q-81 0 -138 -54l-18 23q22 27 39.5 43
t61.5 33.5t100 17.5q43 0 131.5 -2.5t129.5 -2.5q23 0 33.5 5t24.5 25z" id="x1D70B"></path></g><g transform="matrix(.017,-0,0,-.017,10.109,12.137)"><path d="M300 -147l-18 -23q-106 71 -159 185.5t-53 254.5v1q0 139 53 252.5t159 186.5l18 -24q-74 -62 -115.5 -173.5t-41.5 -242.5q0 -130 41.5 -242.5t115.5 -174.5z" id="x28"></path></g><g transform="matrix(.017,-0,0,-.017,15.99,12.137)"><path d="M162 -163h-61v866h61v-866z" id="x7C"></path></g><g transform="matrix(.017,-0,0,-.017,20.461,12.137)"><path d="M716 28l-8 -28h-254l8 28q55 5 66 17.5t5 47.5l-30 145h-232l-74 -138q-21 -43 -11 -54.5t72 -17.5l-6 -28h-235l8 28q53 7 73.5 21t53.5 70l319 540l33 8q6 -43 33 -176l80 -381q10 -49 26.5 -63t72.5 -19zM495 281l-49 264h-2l-149 -264h200z" id="x1D434"></path></g>
<g transform="matrix(.012,-0,0,-.012,32.925,16.212)"><path d="M384 0h-275v27q67 5 81.5 18.5t14.5 68.5v385q0 38 -7.5 47.5t-40.5 10.5l-48 2v24q85 15 178 52v-521q0 -55 14.5 -68.5t82.5 -18.5v-27z" id="x31"></path></g>
<g transform="matrix(.017,-0,0,-.017,39.25,12.137)"><path d="M594 511q0 -122 -171 -157l1 -2q158 -33 158 -159q0 -52 -34.5 -95t-90.5 -65q-76 -33 -217 -33h-223l8 28q63 5 79.5 19t26.5 72l83 426q9 48 -2.5 60t-77.5 17l6 28h259q195 0 195 -139zM499 509q0 59 -37 83t-91 24q-36 0 -51 -9q-17 -9 -22 -44l-35 -195h62
q82 0 128 37t46 104zM481 199q0 71 -48 102.5t-121 31.5h-56l-37 -201q-11 -58 7.5 -77t80.5 -19q76 0 125 44.5t49 118.5z" id="x1D435"></path></g>
<g transform="matrix(.012,-0,0,-.012,49.738,16.212)"><path d="M384 0h-275v27q67 5 81.5 18.5t14.5 68.5v385q0 38 -7.5 47.5t-40.5 10.5l-48 2v24q85 15 178 52v-521q0 -55 14.5 -68.5t82.5 -18.5v-27z" id="x31"></path></g>
<g transform="matrix(.017,-0,0,-.017,56.075,12.137)"><path d="M682 629q-1 -16 -1.5 -72t-2.5 -86l-31 -4q-5 92 -51 129t-139 37q-100 0 -177 -49t-116 -125t-39 -162q0 -122 66 -201t182 -79q83 0 137 42.5t112 128.5l26 -15q-12 -31 -42.5 -88t-45.5 -75q-139 -27 -199 -27q-148 0 -243 81.5t-95 226.5q0 173 129.5 274.5
t325.5 101.5q114 0 204 -38z" id="x1D436"></path></g>
<g transform="matrix(.012,-0,0,-.012,68.062,16.212)"><path d="M384 0h-275v27q67 5 81.5 18.5t14.5 68.5v385q0 38 -7.5 47.5t-40.5 10.5l-48 2v24q85 15 178 52v-521q0 -55 14.5 -68.5t82.5 -18.5v-27z" id="x31"></path></g>
<g transform="matrix(.017,-0,0,-.017,74.4,12.137)"><path d="M780 372q0 -191 -178 -297q-128 -75 -335 -75h-250l7 27q61 7 77 22t28 76l77 398q11 62 0 76.5t-77 22.5l6 28h292q164 0 258.5 -67.5t94.5 -210.5zM678 375q0 119 -73 180.5t-201 61.5q-62 0 -82 -13q-17 -7 -25 -53l-80 -427q-8 -47 -4 -62t26 -20q24 -6 80 -6
q166 0 262.5 95t96.5 244z" id="x1D437"></path></g>
<g transform="matrix(.012,-0,0,-.012,88.05,16.212)"><path d="M384 0h-275v27q67 5 81.5 18.5t14.5 68.5v385q0 38 -7.5 47.5t-40.5 10.5l-48 2v24q85 15 178 52v-521q0 -55 14.5 -68.5t82.5 -18.5v-27z" id="x31"></path></g>
<g transform="matrix(.017,-0,0,-.017,94.375,12.137)"><path d="M162 -163h-61v866h61v-866z" id="x7C"></path></g><g transform="matrix(.017,-0,0,-.017,98.846,12.137)"><path d="M275 270q0 -296 -211 -440l-19 23q75 62 116.5 174t41.5 243t-42 243t-116 173l19 24q211 -144 211 -440z" id="x29"></path></g>
</svg> lies wholly across <svg height="16.15" id="M208" style="vertical-align:-3.27605pt" version="1.1" viewbox="0 0 82.987503 16.15" width="82.987503" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
<g transform="matrix(.017,-0,0,-.017,.062,12.012)"><path d="M162 -163h-61v866h61v-866z" id="x7C"></path></g><g transform="matrix(.017,-0,0,-.017,4.533,12.012)"><path d="M716 28l-8 -28h-254l8 28q55 5 66 17.5t5 47.5l-30 145h-232l-74 -138q-21 -43 -11 -54.5t72 -17.5l-6 -28h-235l8 28q53 7 73.5 21t53.5 70l319 540l33 8q6 -43 33 -176l80 -381q10 -49 26.5 -63t72.5 -19zM495 281l-49 264h-2l-149 -264h200z" id="x1D434"></path></g>
<g transform="matrix(.012,-0,0,-.012,16.988,16.087)"><path d="M384 0h-275v27q67 5 81.5 18.5t14.5 68.5v385q0 38 -7.5 47.5t-40.5 10.5l-48 2v24q85 15 178 52v-521q0 -55 14.5 -68.5t82.5 -18.5v-27z" id="x31"></path></g>
<g transform="matrix(.017,-0,0,-.017,23.325,12.012)"><path d="M594 511q0 -122 -171 -157l1 -2q158 -33 158 -159q0 -52 -34.5 -95t-90.5 -65q-76 -33 -217 -33h-223l8 28q63 5 79.5 19t26.5 72l83 426q9 48 -2.5 60t-77.5 17l6 28h259q195 0 195 -139zM499 509q0 59 -37 83t-91 24q-36 0 -51 -9q-17 -9 -22 -44l-35 -195h62
q82 0 128 37t46 104zM481 199q0 71 -48 102.5t-121 31.5h-56l-37 -201q-11 -58 7.5 -77t80.5 -19q76 0 125 44.5t49 118.5z" id="x1D435"></path></g>
<g transform="matrix(.012,-0,0,-.012,33.812,16.087)"><path d="M384 0h-275v27q67 5 81.5 18.5t14.5 68.5v385q0 38 -7.5 47.5t-40.5 10.5l-48 2v24q85 15 178 52v-521q0 -55 14.5 -68.5t82.5 -18.5v-27z" id="x31"></path></g>
<g transform="matrix(.017,-0,0,-.017,40.15,12.012)"><path d="M682 629q-1 -16 -1.5 -72t-2.5 -86l-31 -4q-5 92 -51 129t-139 37q-100 0 -177 -49t-116 -125t-39 -162q0 -122 66 -201t182 -79q83 0 137 42.5t112 128.5l26 -15q-12 -31 -42.5 -88t-45.5 -75q-139 -27 -199 -27q-148 0 -243 81.5t-95 226.5q0 173 129.5 274.5
t325.5 101.5q114 0 204 -38z" id="x1D436"></path></g>
<g transform="matrix(.012,-0,0,-.012,52.138,16.087)"><path d="M384 0h-275v27q67 5 81.5 18.5t14.5 68.5v385q0 38 -7.5 47.5t-40.5 10.5l-48 2v24q85 15 178 52v-521q0 -55 14.5 -68.5t82.5 -18.5v-27z" id="x31"></path></g>
<g transform="matrix(.017,-0,0,-.017,58.462,12.012)"><path d="M780 372q0 -191 -178 -297q-128 -75 -335 -75h-250l7 27q61 7 77 22t28 76l77 398q11 62 0 76.5t-77 22.5l6 28h292q164 0 258.5 -67.5t94.5 -210.5zM678 375q0 119 -73 180.5t-201 61.5q-62 0 -82 -13q-17 -7 -25 -53l-80 -427q-8 -47 -4 -62t26 -20q24 -6 80 -6
q166 0 262.5 95t96.5 244z" id="x1D437"></path></g>
<g transform="matrix(.012,-0,0,-.012,72.112,16.087)"><path d="M384 0h-275v27q67 5 81.5 18.5t14.5 68.5v385q0 38 -7.5 47.5t-40.5 10.5l-48 2v24q85 15 178 52v-521q0 -55 14.5 -68.5t82.5 -18.5v-27z" id="x31"></path></g>
<g transform="matrix(.017,-0,0,-.017,78.45,12.012)"><path d="M162 -163h-61v866h61v-866z" id="x7C"></path></g>
</svg> and <svg height="16.15" id="M209" style="vertical-align:-3.27605pt" version="1.1" viewbox="0 0 82.987503 16.15" width="82.987503" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
<g transform="matrix(.017,-0,0,-.017,.062,12.012)"><path d="M162 -163h-61v866h61v-866z" id="x7C"></path></g><g transform="matrix(.017,-0,0,-.017,4.533,12.012)"><path d="M716 28l-8 -28h-254l8 28q55 5 66 17.5t5 47.5l-30 145h-232l-74 -138q-21 -43 -11 -54.5t72 -17.5l-6 -28h-235l8 28q53 7 73.5 21t53.5 70l319 540l33 8q6 -43 33 -176l80 -381q10 -49 26.5 -63t72.5 -19zM495 281l-49 264h-2l-149 -264h200z" id="x1D434"></path></g>
<g transform="matrix(.012,-0,0,-.012,16.988,16.087)"><path d="M412 140l28 -9q0 -2 -35 -131h-373v23q112 112 161 170q59 70 92 127t33 115q0 63 -31 98t-86 35q-75 0 -137 -93l-22 20l57 81q55 59 135 59q69 0 118.5 -46.5t49.5 -122.5q0 -62 -29.5 -114t-102.5 -130l-141 -149h186q42 0 58.5 10.5t38.5 56.5z" id="x32"></path></g>
<g transform="matrix(.017,-0,0,-.017,23.325,12.012)"><path d="M594 511q0 -122 -171 -157l1 -2q158 -33 158 -159q0 -52 -34.5 -95t-90.5 -65q-76 -33 -217 -33h-223l8 28q63 5 79.5 19t26.5 72l83 426q9 48 -2.5 60t-77.5 17l6 28h259q195 0 195 -139zM499 509q0 59 -37 83t-91 24q-36 0 -51 -9q-17 -9 -22 -44l-35 -195h62
q82 0 128 37t46 104zM481 199q0 71 -48 102.5t-121 31.5h-56l-37 -201q-11 -58 7.5 -77t80.5 -19q76 0 125 44.5t49 118.5z" id="x1D435"></path></g>
<g transform="matrix(.012,-0,0,-.012,33.812,16.087)"><path d="M412 140l28 -9q0 -2 -35 -131h-373v23q112 112 161 170q59 70 92 127t33 115q0 63 -31 98t-86 35q-75 0 -137 -93l-22 20l57 81q55 59 135 59q69 0 118.5 -46.5t49.5 -122.5q0 -62 -29.5 -114t-102.5 -130l-141 -149h186q42 0 58.5 10.5t38.5 56.5z" id="x32"></path></g>
<g transform="matrix(.017,-0,0,-.017,40.15,12.012)"><path d="M682 629q-1 -16 -1.5 -72t-2.5 -86l-31 -4q-5 92 -51 129t-139 37q-100 0 -177 -49t-116 -125t-39 -162q0 -122 66 -201t182 -79q83 0 137 42.5t112 128.5l26 -15q-12 -31 -42.5 -88t-45.5 -75q-139 -27 -199 -27q-148 0 -243 81.5t-95 226.5q0 173 129.5 274.5
t325.5 101.5q114 0 204 -38z" id="x1D436"></path></g>
<g transform="matrix(.012,-0,0,-.012,52.138,16.087)"><path d="M412 140l28 -9q0 -2 -35 -131h-373v23q112 112 161 170q59 70 92 127t33 115q0 63 -31 98t-86 35q-75 0 -137 -93l-22 20l57 81q55 59 135 59q69 0 118.5 -46.5t49.5 -122.5q0 -62 -29.5 -114t-102.5 -130l-141 -149h186q42 0 58.5 10.5t38.5 56.5z" id="x32"></path></g>
<g transform="matrix(.017,-0,0,-.017,58.462,12.012)"><path d="M780 372q0 -191 -178 -297q-128 -75 -335 -75h-250l7 27q61 7 77 22t28 76l77 398q11 62 0 76.5t-77 22.5l6 28h292q164 0 258.5 -67.5t94.5 -210.5zM678 375q0 119 -73 180.5t-201 61.5q-62 0 -82 -13q-17 -7 -25 -53l-80 -427q-8 -47 -4 -62t26 -20q24 -6 80 -6
q166 0 262.5 95t96.5 244z" id="x1D437"></path></g>
<g transform="matrix(.012,-0,0,-.012,72.112,16.087)"><path d="M412 140l28 -9q0 -2 -35 -131h-373v23q112 112 161 170q59 70 92 127t33 115q0 63 -31 98t-86 35q-75 0 -137 -93l-22 20l57 81q55 59 135 59q69 0 118.5 -46.5t49.5 -122.5q0 -62 -29.5 -114t-102.5 -130l-141 -149h186q42 0 58.5 10.5t38.5 56.5z" id="x32"></path></g>
<g transform="matrix(.017,-0,0,-.017,78.45,12.012)"><path d="M162 -163h-61v866h61v-866z" id="x7C"></path></g>
</svg>.</table>

Discrete Dynamics in Nature and Society

fig7

Figure 7

Figure 7: Horseshoe Chaos in a 3D Neural Network with Different Activation Functions 