00001
00006
00007
00008
00009
00010
00011
00012
00013
00014
00015
00016
00017
00018
00019
00020
00021
00022
00023
00024
00025
00026
00027
00028
00029
00030
00031
00032
00033
00034
00035
00036
00037
00038 #ifndef MBEDTLS_BN_MUL_H
00039 #define MBEDTLS_BN_MUL_H
00040
00041 #if !defined(MBEDTLS_CONFIG_FILE)
00042 #include "config.h"
00043 #else
00044 #include MBEDTLS_CONFIG_FILE
00045 #endif
00046
00047 #include "bignum.h"
00048
00049 #if defined(MBEDTLS_HAVE_ASM)
00050
00051 #ifndef asm
00052 #define asm __asm
00053 #endif
00054
00055
00056 #if defined(__GNUC__) && \
00057 ( !defined(__ARMCC_VERSION) || __ARMCC_VERSION >= 6000000 )
00058
00059
00060
00061
00062
00063
00064
00065
00066
00067
00068
00069
00070
00071 #if defined(__GNUC__) && __GNUC__ < 5 && defined(__PIC__)
00072 #define MULADDC_CANNOT_USE_EBX
00073 #endif
00074
00075
00076
00077
00078
00079
00080
00081 #if defined(__i386__) && defined(__OPTIMIZE__) && !defined(MULADDC_CANNOT_USE_EBX)
00082
00083 #define MULADDC_INIT \
00084 asm( \
00085 "movl %%ebx, %0 \n\t" \
00086 "movl %5, %%esi \n\t" \
00087 "movl %6, %%edi \n\t" \
00088 "movl %7, %%ecx \n\t" \
00089 "movl %8, %%ebx \n\t"
00090
00091 #define MULADDC_CORE \
00092 "lodsl \n\t" \
00093 "mull %%ebx \n\t" \
00094 "addl %%ecx, %%eax \n\t" \
00095 "adcl $0, %%edx \n\t" \
00096 "addl (%%edi), %%eax \n\t" \
00097 "adcl $0, %%edx \n\t" \
00098 "movl %%edx, %%ecx \n\t" \
00099 "stosl \n\t"
00100
00101 #if defined(MBEDTLS_HAVE_SSE2)
00102
00103 #define MULADDC_HUIT \
00104 "movd %%ecx, %%mm1 \n\t" \
00105 "movd %%ebx, %%mm0 \n\t" \
00106 "movd (%%edi), %%mm3 \n\t" \
00107 "paddq %%mm3, %%mm1 \n\t" \
00108 "movd (%%esi), %%mm2 \n\t" \
00109 "pmuludq %%mm0, %%mm2 \n\t" \
00110 "movd 4(%%esi), %%mm4 \n\t" \
00111 "pmuludq %%mm0, %%mm4 \n\t" \
00112 "movd 8(%%esi), %%mm6 \n\t" \
00113 "pmuludq %%mm0, %%mm6 \n\t" \
00114 "movd 12(%%esi), %%mm7 \n\t" \
00115 "pmuludq %%mm0, %%mm7 \n\t" \
00116 "paddq %%mm2, %%mm1 \n\t" \
00117 "movd 4(%%edi), %%mm3 \n\t" \
00118 "paddq %%mm4, %%mm3 \n\t" \
00119 "movd 8(%%edi), %%mm5 \n\t" \
00120 "paddq %%mm6, %%mm5 \n\t" \
00121 "movd 12(%%edi), %%mm4 \n\t" \
00122 "paddq %%mm4, %%mm7 \n\t" \
00123 "movd %%mm1, (%%edi) \n\t" \
00124 "movd 16(%%esi), %%mm2 \n\t" \
00125 "pmuludq %%mm0, %%mm2 \n\t" \
00126 "psrlq $32, %%mm1 \n\t" \
00127 "movd 20(%%esi), %%mm4 \n\t" \
00128 "pmuludq %%mm0, %%mm4 \n\t" \
00129 "paddq %%mm3, %%mm1 \n\t" \
00130 "movd 24(%%esi), %%mm6 \n\t" \
00131 "pmuludq %%mm0, %%mm6 \n\t" \
00132 "movd %%mm1, 4(%%edi) \n\t" \
00133 "psrlq $32, %%mm1 \n\t" \
00134 "movd 28(%%esi), %%mm3 \n\t" \
00135 "pmuludq %%mm0, %%mm3 \n\t" \
00136 "paddq %%mm5, %%mm1 \n\t" \
00137 "movd 16(%%edi), %%mm5 \n\t" \
00138 "paddq %%mm5, %%mm2 \n\t" \
00139 "movd %%mm1, 8(%%edi) \n\t" \
00140 "psrlq $32, %%mm1 \n\t" \
00141 "paddq %%mm7, %%mm1 \n\t" \
00142 "movd 20(%%edi), %%mm5 \n\t" \
00143 "paddq %%mm5, %%mm4 \n\t" \
00144 "movd %%mm1, 12(%%edi) \n\t" \
00145 "psrlq $32, %%mm1 \n\t" \
00146 "paddq %%mm2, %%mm1 \n\t" \
00147 "movd 24(%%edi), %%mm5 \n\t" \
00148 "paddq %%mm5, %%mm6 \n\t" \
00149 "movd %%mm1, 16(%%edi) \n\t" \
00150 "psrlq $32, %%mm1 \n\t" \
00151 "paddq %%mm4, %%mm1 \n\t" \
00152 "movd 28(%%edi), %%mm5 \n\t" \
00153 "paddq %%mm5, %%mm3 \n\t" \
00154 "movd %%mm1, 20(%%edi) \n\t" \
00155 "psrlq $32, %%mm1 \n\t" \
00156 "paddq %%mm6, %%mm1 \n\t" \
00157 "movd %%mm1, 24(%%edi) \n\t" \
00158 "psrlq $32, %%mm1 \n\t" \
00159 "paddq %%mm3, %%mm1 \n\t" \
00160 "movd %%mm1, 28(%%edi) \n\t" \
00161 "addl $32, %%edi \n\t" \
00162 "addl $32, %%esi \n\t" \
00163 "psrlq $32, %%mm1 \n\t" \
00164 "movd %%mm1, %%ecx \n\t"
00165
00166 #define MULADDC_STOP \
00167 "emms \n\t" \
00168 "movl %4, %%ebx \n\t" \
00169 "movl %%ecx, %1 \n\t" \
00170 "movl %%edi, %2 \n\t" \
00171 "movl %%esi, %3 \n\t" \
00172 : "=m" (t), "=m" (c), "=m" (d), "=m" (s) \
00173 : "m" (t), "m" (s), "m" (d), "m" (c), "m" (b) \
00174 : "eax", "ebx", "ecx", "edx", "esi", "edi" \
00175 );
00176
00177 #else
00178
00179 #define MULADDC_STOP \
00180 "movl %4, %%ebx \n\t" \
00181 "movl %%ecx, %1 \n\t" \
00182 "movl %%edi, %2 \n\t" \
00183 "movl %%esi, %3 \n\t" \
00184 : "=m" (t), "=m" (c), "=m" (d), "=m" (s) \
00185 : "m" (t), "m" (s), "m" (d), "m" (c), "m" (b) \
00186 : "eax", "ebx", "ecx", "edx", "esi", "edi" \
00187 );
00188 #endif
00189 #endif
00190
00191 #if defined(__amd64__) || defined (__x86_64__)
00192
00193 #define MULADDC_INIT \
00194 asm( \
00195 "xorq %%r8, %%r8\n"
00196
00197 #define MULADDC_CORE \
00198 "movq (%%rsi), %%rax\n" \
00199 "mulq %%rbx\n" \
00200 "addq $8, %%rsi\n" \
00201 "addq %%rcx, %%rax\n" \
00202 "movq %%r8, %%rcx\n" \
00203 "adcq $0, %%rdx\n" \
00204 "nop \n" \
00205 "addq %%rax, (%%rdi)\n" \
00206 "adcq %%rdx, %%rcx\n" \
00207 "addq $8, %%rdi\n"
00208
00209 #define MULADDC_STOP \
00210 : "+c" (c), "+D" (d), "+S" (s) \
00211 : "b" (b) \
00212 : "rax", "rdx", "r8" \
00213 );
00214
00215 #endif
00216
00217 #if defined(__mc68020__) || defined(__mcpu32__)
00218
00219 #define MULADDC_INIT \
00220 asm( \
00221 "movl %3, %%a2 \n\t" \
00222 "movl %4, %%a3 \n\t" \
00223 "movl %5, %%d3 \n\t" \
00224 "movl %6, %%d2 \n\t" \
00225 "moveq #0, %%d0 \n\t"
00226
00227 #define MULADDC_CORE \
00228 "movel %%a2@+, %%d1 \n\t" \
00229 "mulul %%d2, %%d4:%%d1 \n\t" \
00230 "addl %%d3, %%d1 \n\t" \
00231 "addxl %%d0, %%d4 \n\t" \
00232 "moveq #0, %%d3 \n\t" \
00233 "addl %%d1, %%a3@+ \n\t" \
00234 "addxl %%d4, %%d3 \n\t"
00235
00236 #define MULADDC_STOP \
00237 "movl %%d3, %0 \n\t" \
00238 "movl %%a3, %1 \n\t" \
00239 "movl %%a2, %2 \n\t" \
00240 : "=m" (c), "=m" (d), "=m" (s) \
00241 : "m" (s), "m" (d), "m" (c), "m" (b) \
00242 : "d0", "d1", "d2", "d3", "d4", "a2", "a3" \
00243 );
00244
00245 #define MULADDC_HUIT \
00246 "movel %%a2@+, %%d1 \n\t" \
00247 "mulul %%d2, %%d4:%%d1 \n\t" \
00248 "addxl %%d3, %%d1 \n\t" \
00249 "addxl %%d0, %%d4 \n\t" \
00250 "addl %%d1, %%a3@+ \n\t" \
00251 "movel %%a2@+, %%d1 \n\t" \
00252 "mulul %%d2, %%d3:%%d1 \n\t" \
00253 "addxl %%d4, %%d1 \n\t" \
00254 "addxl %%d0, %%d3 \n\t" \
00255 "addl %%d1, %%a3@+ \n\t" \
00256 "movel %%a2@+, %%d1 \n\t" \
00257 "mulul %%d2, %%d4:%%d1 \n\t" \
00258 "addxl %%d3, %%d1 \n\t" \
00259 "addxl %%d0, %%d4 \n\t" \
00260 "addl %%d1, %%a3@+ \n\t" \
00261 "movel %%a2@+, %%d1 \n\t" \
00262 "mulul %%d2, %%d3:%%d1 \n\t" \
00263 "addxl %%d4, %%d1 \n\t" \
00264 "addxl %%d0, %%d3 \n\t" \
00265 "addl %%d1, %%a3@+ \n\t" \
00266 "movel %%a2@+, %%d1 \n\t" \
00267 "mulul %%d2, %%d4:%%d1 \n\t" \
00268 "addxl %%d3, %%d1 \n\t" \
00269 "addxl %%d0, %%d4 \n\t" \
00270 "addl %%d1, %%a3@+ \n\t" \
00271 "movel %%a2@+, %%d1 \n\t" \
00272 "mulul %%d2, %%d3:%%d1 \n\t" \
00273 "addxl %%d4, %%d1 \n\t" \
00274 "addxl %%d0, %%d3 \n\t" \
00275 "addl %%d1, %%a3@+ \n\t" \
00276 "movel %%a2@+, %%d1 \n\t" \
00277 "mulul %%d2, %%d4:%%d1 \n\t" \
00278 "addxl %%d3, %%d1 \n\t" \
00279 "addxl %%d0, %%d4 \n\t" \
00280 "addl %%d1, %%a3@+ \n\t" \
00281 "movel %%a2@+, %%d1 \n\t" \
00282 "mulul %%d2, %%d3:%%d1 \n\t" \
00283 "addxl %%d4, %%d1 \n\t" \
00284 "addxl %%d0, %%d3 \n\t" \
00285 "addl %%d1, %%a3@+ \n\t" \
00286 "addxl %%d0, %%d3 \n\t"
00287
00288 #endif
00289
00290 #if defined(__powerpc64__) || defined(__ppc64__)
00291
00292 #if defined(__MACH__) && defined(__APPLE__)
00293
00294 #define MULADDC_INIT \
00295 asm( \
00296 "ld r3, %3 \n\t" \
00297 "ld r4, %4 \n\t" \
00298 "ld r5, %5 \n\t" \
00299 "ld r6, %6 \n\t" \
00300 "addi r3, r3, -8 \n\t" \
00301 "addi r4, r4, -8 \n\t" \
00302 "addic r5, r5, 0 \n\t"
00303
00304 #define MULADDC_CORE \
00305 "ldu r7, 8(r3) \n\t" \
00306 "mulld r8, r7, r6 \n\t" \
00307 "mulhdu r9, r7, r6 \n\t" \
00308 "adde r8, r8, r5 \n\t" \
00309 "ld r7, 8(r4) \n\t" \
00310 "addze r5, r9 \n\t" \
00311 "addc r8, r8, r7 \n\t" \
00312 "stdu r8, 8(r4) \n\t"
00313
00314 #define MULADDC_STOP \
00315 "addze r5, r5 \n\t" \
00316 "addi r4, r4, 8 \n\t" \
00317 "addi r3, r3, 8 \n\t" \
00318 "std r5, %0 \n\t" \
00319 "std r4, %1 \n\t" \
00320 "std r3, %2 \n\t" \
00321 : "=m" (c), "=m" (d), "=m" (s) \
00322 : "m" (s), "m" (d), "m" (c), "m" (b) \
00323 : "r3", "r4", "r5", "r6", "r7", "r8", "r9" \
00324 );
00325
00326
00327 #else
00328
00329 #define MULADDC_INIT \
00330 asm( \
00331 "ld %%r3, %3 \n\t" \
00332 "ld %%r4, %4 \n\t" \
00333 "ld %%r5, %5 \n\t" \
00334 "ld %%r6, %6 \n\t" \
00335 "addi %%r3, %%r3, -8 \n\t" \
00336 "addi %%r4, %%r4, -8 \n\t" \
00337 "addic %%r5, %%r5, 0 \n\t"
00338
00339 #define MULADDC_CORE \
00340 "ldu %%r7, 8(%%r3) \n\t" \
00341 "mulld %%r8, %%r7, %%r6 \n\t" \
00342 "mulhdu %%r9, %%r7, %%r6 \n\t" \
00343 "adde %%r8, %%r8, %%r5 \n\t" \
00344 "ld %%r7, 8(%%r4) \n\t" \
00345 "addze %%r5, %%r9 \n\t" \
00346 "addc %%r8, %%r8, %%r7 \n\t" \
00347 "stdu %%r8, 8(%%r4) \n\t"
00348
00349 #define MULADDC_STOP \
00350 "addze %%r5, %%r5 \n\t" \
00351 "addi %%r4, %%r4, 8 \n\t" \
00352 "addi %%r3, %%r3, 8 \n\t" \
00353 "std %%r5, %0 \n\t" \
00354 "std %%r4, %1 \n\t" \
00355 "std %%r3, %2 \n\t" \
00356 : "=m" (c), "=m" (d), "=m" (s) \
00357 : "m" (s), "m" (d), "m" (c), "m" (b) \
00358 : "r3", "r4", "r5", "r6", "r7", "r8", "r9" \
00359 );
00360
00361 #endif
00362
00363 #elif defined(__powerpc__) || defined(__ppc__)
00364
00365 #if defined(__MACH__) && defined(__APPLE__)
00366
00367 #define MULADDC_INIT \
00368 asm( \
00369 "lwz r3, %3 \n\t" \
00370 "lwz r4, %4 \n\t" \
00371 "lwz r5, %5 \n\t" \
00372 "lwz r6, %6 \n\t" \
00373 "addi r3, r3, -4 \n\t" \
00374 "addi r4, r4, -4 \n\t" \
00375 "addic r5, r5, 0 \n\t"
00376
00377 #define MULADDC_CORE \
00378 "lwzu r7, 4(r3) \n\t" \
00379 "mullw r8, r7, r6 \n\t" \
00380 "mulhwu r9, r7, r6 \n\t" \
00381 "adde r8, r8, r5 \n\t" \
00382 "lwz r7, 4(r4) \n\t" \
00383 "addze r5, r9 \n\t" \
00384 "addc r8, r8, r7 \n\t" \
00385 "stwu r8, 4(r4) \n\t"
00386
00387 #define MULADDC_STOP \
00388 "addze r5, r5 \n\t" \
00389 "addi r4, r4, 4 \n\t" \
00390 "addi r3, r3, 4 \n\t" \
00391 "stw r5, %0 \n\t" \
00392 "stw r4, %1 \n\t" \
00393 "stw r3, %2 \n\t" \
00394 : "=m" (c), "=m" (d), "=m" (s) \
00395 : "m" (s), "m" (d), "m" (c), "m" (b) \
00396 : "r3", "r4", "r5", "r6", "r7", "r8", "r9" \
00397 );
00398
00399 #else
00400
00401 #define MULADDC_INIT \
00402 asm( \
00403 "lwz %%r3, %3 \n\t" \
00404 "lwz %%r4, %4 \n\t" \
00405 "lwz %%r5, %5 \n\t" \
00406 "lwz %%r6, %6 \n\t" \
00407 "addi %%r3, %%r3, -4 \n\t" \
00408 "addi %%r4, %%r4, -4 \n\t" \
00409 "addic %%r5, %%r5, 0 \n\t"
00410
00411 #define MULADDC_CORE \
00412 "lwzu %%r7, 4(%%r3) \n\t" \
00413 "mullw %%r8, %%r7, %%r6 \n\t" \
00414 "mulhwu %%r9, %%r7, %%r6 \n\t" \
00415 "adde %%r8, %%r8, %%r5 \n\t" \
00416 "lwz %%r7, 4(%%r4) \n\t" \
00417 "addze %%r5, %%r9 \n\t" \
00418 "addc %%r8, %%r8, %%r7 \n\t" \
00419 "stwu %%r8, 4(%%r4) \n\t"
00420
00421 #define MULADDC_STOP \
00422 "addze %%r5, %%r5 \n\t" \
00423 "addi %%r4, %%r4, 4 \n\t" \
00424 "addi %%r3, %%r3, 4 \n\t" \
00425 "stw %%r5, %0 \n\t" \
00426 "stw %%r4, %1 \n\t" \
00427 "stw %%r3, %2 \n\t" \
00428 : "=m" (c), "=m" (d), "=m" (s) \
00429 : "m" (s), "m" (d), "m" (c), "m" (b) \
00430 : "r3", "r4", "r5", "r6", "r7", "r8", "r9" \
00431 );
00432
00433 #endif
00434
00435 #endif
00436
00437
00438
00439
00440
00441 #if 0 && defined(__sparc__)
00442 #if defined(__sparc64__)
00443
00444 #define MULADDC_INIT \
00445 asm( \
00446 "ldx %3, %%o0 \n\t" \
00447 "ldx %4, %%o1 \n\t" \
00448 "ld %5, %%o2 \n\t" \
00449 "ld %6, %%o3 \n\t"
00450
00451 #define MULADDC_CORE \
00452 "ld [%%o0], %%o4 \n\t" \
00453 "inc 4, %%o0 \n\t" \
00454 "ld [%%o1], %%o5 \n\t" \
00455 "umul %%o3, %%o4, %%o4 \n\t" \
00456 "addcc %%o4, %%o2, %%o4 \n\t" \
00457 "rd %%y, %%g1 \n\t" \
00458 "addx %%g1, 0, %%g1 \n\t" \
00459 "addcc %%o4, %%o5, %%o4 \n\t" \
00460 "st %%o4, [%%o1] \n\t" \
00461 "addx %%g1, 0, %%o2 \n\t" \
00462 "inc 4, %%o1 \n\t"
00463
00464 #define MULADDC_STOP \
00465 "st %%o2, %0 \n\t" \
00466 "stx %%o1, %1 \n\t" \
00467 "stx %%o0, %2 \n\t" \
00468 : "=m" (c), "=m" (d), "=m" (s) \
00469 : "m" (s), "m" (d), "m" (c), "m" (b) \
00470 : "g1", "o0", "o1", "o2", "o3", "o4", \
00471 "o5" \
00472 );
00473
00474 #else
00475
00476 #define MULADDC_INIT \
00477 asm( \
00478 "ld %3, %%o0 \n\t" \
00479 "ld %4, %%o1 \n\t" \
00480 "ld %5, %%o2 \n\t" \
00481 "ld %6, %%o3 \n\t"
00482
00483 #define MULADDC_CORE \
00484 "ld [%%o0], %%o4 \n\t" \
00485 "inc 4, %%o0 \n\t" \
00486 "ld [%%o1], %%o5 \n\t" \
00487 "umul %%o3, %%o4, %%o4 \n\t" \
00488 "addcc %%o4, %%o2, %%o4 \n\t" \
00489 "rd %%y, %%g1 \n\t" \
00490 "addx %%g1, 0, %%g1 \n\t" \
00491 "addcc %%o4, %%o5, %%o4 \n\t" \
00492 "st %%o4, [%%o1] \n\t" \
00493 "addx %%g1, 0, %%o2 \n\t" \
00494 "inc 4, %%o1 \n\t"
00495
00496 #define MULADDC_STOP \
00497 "st %%o2, %0 \n\t" \
00498 "st %%o1, %1 \n\t" \
00499 "st %%o0, %2 \n\t" \
00500 : "=m" (c), "=m" (d), "=m" (s) \
00501 : "m" (s), "m" (d), "m" (c), "m" (b) \
00502 : "g1", "o0", "o1", "o2", "o3", "o4", \
00503 "o5" \
00504 );
00505
00506 #endif
00507 #endif
00508
00509 #if defined(__microblaze__) || defined(microblaze)
00510
00511 #define MULADDC_INIT \
00512 asm( \
00513 "lwi r3, %3 \n\t" \
00514 "lwi r4, %4 \n\t" \
00515 "lwi r5, %5 \n\t" \
00516 "lwi r6, %6 \n\t" \
00517 "andi r7, r6, 0xffff \n\t" \
00518 "bsrli r6, r6, 16 \n\t"
00519
00520 #define MULADDC_CORE \
00521 "lhui r8, r3, 0 \n\t" \
00522 "addi r3, r3, 2 \n\t" \
00523 "lhui r9, r3, 0 \n\t" \
00524 "addi r3, r3, 2 \n\t" \
00525 "mul r10, r9, r6 \n\t" \
00526 "mul r11, r8, r7 \n\t" \
00527 "mul r12, r9, r7 \n\t" \
00528 "mul r13, r8, r6 \n\t" \
00529 "bsrli r8, r10, 16 \n\t" \
00530 "bsrli r9, r11, 16 \n\t" \
00531 "add r13, r13, r8 \n\t" \
00532 "add r13, r13, r9 \n\t" \
00533 "bslli r10, r10, 16 \n\t" \
00534 "bslli r11, r11, 16 \n\t" \
00535 "add r12, r12, r10 \n\t" \
00536 "addc r13, r13, r0 \n\t" \
00537 "add r12, r12, r11 \n\t" \
00538 "addc r13, r13, r0 \n\t" \
00539 "lwi r10, r4, 0 \n\t" \
00540 "add r12, r12, r10 \n\t" \
00541 "addc r13, r13, r0 \n\t" \
00542 "add r12, r12, r5 \n\t" \
00543 "addc r5, r13, r0 \n\t" \
00544 "swi r12, r4, 0 \n\t" \
00545 "addi r4, r4, 4 \n\t"
00546
00547 #define MULADDC_STOP \
00548 "swi r5, %0 \n\t" \
00549 "swi r4, %1 \n\t" \
00550 "swi r3, %2 \n\t" \
00551 : "=m" (c), "=m" (d), "=m" (s) \
00552 : "m" (s), "m" (d), "m" (c), "m" (b) \
00553 : "r3", "r4", "r5", "r6", "r7", "r8", \
00554 "r9", "r10", "r11", "r12", "r13" \
00555 );
00556
00557 #endif
00558
00559 #if defined(__tricore__)
00560
00561 #define MULADDC_INIT \
00562 asm( \
00563 "ld.a %%a2, %3 \n\t" \
00564 "ld.a %%a3, %4 \n\t" \
00565 "ld.w %%d4, %5 \n\t" \
00566 "ld.w %%d1, %6 \n\t" \
00567 "xor %%d5, %%d5 \n\t"
00568
00569 #define MULADDC_CORE \
00570 "ld.w %%d0, [%%a2+] \n\t" \
00571 "madd.u %%e2, %%e4, %%d0, %%d1 \n\t" \
00572 "ld.w %%d0, [%%a3] \n\t" \
00573 "addx %%d2, %%d2, %%d0 \n\t" \
00574 "addc %%d3, %%d3, 0 \n\t" \
00575 "mov %%d4, %%d3 \n\t" \
00576 "st.w [%%a3+], %%d2 \n\t"
00577
00578 #define MULADDC_STOP \
00579 "st.w %0, %%d4 \n\t" \
00580 "st.a %1, %%a3 \n\t" \
00581 "st.a %2, %%a2 \n\t" \
00582 : "=m" (c), "=m" (d), "=m" (s) \
00583 : "m" (s), "m" (d), "m" (c), "m" (b) \
00584 : "d0", "d1", "e2", "d4", "a2", "a3" \
00585 );
00586
00587 #endif
00588
00589
00590
00591
00592
00593
00594
00595
00596
00597
00598
00599
00600
00601 #if defined(__GNUC__) && !defined(__OPTIMIZE__)
00602 #define MULADDC_CANNOT_USE_R7
00603 #endif
00604
00605 #if defined(__arm__) && !defined(MULADDC_CANNOT_USE_R7)
00606
00607 #if defined(__thumb__) && !defined(__thumb2__)
00608
00609 #define MULADDC_INIT \
00610 asm( \
00611 "ldr r0, %3 \n\t" \
00612 "ldr r1, %4 \n\t" \
00613 "ldr r2, %5 \n\t" \
00614 "ldr r3, %6 \n\t" \
00615 "lsr r7, r3, #16 \n\t" \
00616 "mov r9, r7 \n\t" \
00617 "lsl r7, r3, #16 \n\t" \
00618 "lsr r7, r7, #16 \n\t" \
00619 "mov r8, r7 \n\t"
00620
00621 #define MULADDC_CORE \
00622 "ldmia r0!, {r6} \n\t" \
00623 "lsr r7, r6, #16 \n\t" \
00624 "lsl r6, r6, #16 \n\t" \
00625 "lsr r6, r6, #16 \n\t" \
00626 "mov r4, r8 \n\t" \
00627 "mul r4, r6 \n\t" \
00628 "mov r3, r9 \n\t" \
00629 "mul r6, r3 \n\t" \
00630 "mov r5, r9 \n\t" \
00631 "mul r5, r7 \n\t" \
00632 "mov r3, r8 \n\t" \
00633 "mul r7, r3 \n\t" \
00634 "lsr r3, r6, #16 \n\t" \
00635 "add r5, r5, r3 \n\t" \
00636 "lsr r3, r7, #16 \n\t" \
00637 "add r5, r5, r3 \n\t" \
00638 "add r4, r4, r2 \n\t" \
00639 "mov r2, #0 \n\t" \
00640 "adc r5, r2 \n\t" \
00641 "lsl r3, r6, #16 \n\t" \
00642 "add r4, r4, r3 \n\t" \
00643 "adc r5, r2 \n\t" \
00644 "lsl r3, r7, #16 \n\t" \
00645 "add r4, r4, r3 \n\t" \
00646 "adc r5, r2 \n\t" \
00647 "ldr r3, [r1] \n\t" \
00648 "add r4, r4, r3 \n\t" \
00649 "adc r2, r5 \n\t" \
00650 "stmia r1!, {r4} \n\t"
00651
00652 #define MULADDC_STOP \
00653 "str r2, %0 \n\t" \
00654 "str r1, %1 \n\t" \
00655 "str r0, %2 \n\t" \
00656 : "=m" (c), "=m" (d), "=m" (s) \
00657 : "m" (s), "m" (d), "m" (c), "m" (b) \
00658 : "r0", "r1", "r2", "r3", "r4", "r5", \
00659 "r6", "r7", "r8", "r9", "cc" \
00660 );
00661
00662 #else
00663
00664 #define MULADDC_INIT \
00665 asm( \
00666 "ldr r0, %3 \n\t" \
00667 "ldr r1, %4 \n\t" \
00668 "ldr r2, %5 \n\t" \
00669 "ldr r3, %6 \n\t"
00670
00671 #define MULADDC_CORE \
00672 "ldr r4, [r0], #4 \n\t" \
00673 "mov r5, #0 \n\t" \
00674 "ldr r6, [r1] \n\t" \
00675 "umlal r2, r5, r3, r4 \n\t" \
00676 "adds r7, r6, r2 \n\t" \
00677 "adc r2, r5, #0 \n\t" \
00678 "str r7, [r1], #4 \n\t"
00679
00680 #define MULADDC_STOP \
00681 "str r2, %0 \n\t" \
00682 "str r1, %1 \n\t" \
00683 "str r0, %2 \n\t" \
00684 : "=m" (c), "=m" (d), "=m" (s) \
00685 : "m" (s), "m" (d), "m" (c), "m" (b) \
00686 : "r0", "r1", "r2", "r3", "r4", "r5", \
00687 "r6", "r7", "cc" \
00688 );
00689
00690 #endif
00691
00692 #endif
00693
00694 #if defined(__alpha__)
00695
00696 #define MULADDC_INIT \
00697 asm( \
00698 "ldq $1, %3 \n\t" \
00699 "ldq $2, %4 \n\t" \
00700 "ldq $3, %5 \n\t" \
00701 "ldq $4, %6 \n\t"
00702
00703 #define MULADDC_CORE \
00704 "ldq $6, 0($1) \n\t" \
00705 "addq $1, 8, $1 \n\t" \
00706 "mulq $6, $4, $7 \n\t" \
00707 "umulh $6, $4, $6 \n\t" \
00708 "addq $7, $3, $7 \n\t" \
00709 "cmpult $7, $3, $3 \n\t" \
00710 "ldq $5, 0($2) \n\t" \
00711 "addq $7, $5, $7 \n\t" \
00712 "cmpult $7, $5, $5 \n\t" \
00713 "stq $7, 0($2) \n\t" \
00714 "addq $2, 8, $2 \n\t" \
00715 "addq $6, $3, $3 \n\t" \
00716 "addq $5, $3, $3 \n\t"
00717
00718 #define MULADDC_STOP \
00719 "stq $3, %0 \n\t" \
00720 "stq $2, %1 \n\t" \
00721 "stq $1, %2 \n\t" \
00722 : "=m" (c), "=m" (d), "=m" (s) \
00723 : "m" (s), "m" (d), "m" (c), "m" (b) \
00724 : "$1", "$2", "$3", "$4", "$5", "$6", "$7" \
00725 );
00726 #endif
00727
00728 #if defined(__mips__) && !defined(__mips64)
00729
00730 #define MULADDC_INIT \
00731 asm( \
00732 "lw $10, %3 \n\t" \
00733 "lw $11, %4 \n\t" \
00734 "lw $12, %5 \n\t" \
00735 "lw $13, %6 \n\t"
00736
00737 #define MULADDC_CORE \
00738 "lw $14, 0($10) \n\t" \
00739 "multu $13, $14 \n\t" \
00740 "addi $10, $10, 4 \n\t" \
00741 "mflo $14 \n\t" \
00742 "mfhi $9 \n\t" \
00743 "addu $14, $12, $14 \n\t" \
00744 "lw $15, 0($11) \n\t" \
00745 "sltu $12, $14, $12 \n\t" \
00746 "addu $15, $14, $15 \n\t" \
00747 "sltu $14, $15, $14 \n\t" \
00748 "addu $12, $12, $9 \n\t" \
00749 "sw $15, 0($11) \n\t" \
00750 "addu $12, $12, $14 \n\t" \
00751 "addi $11, $11, 4 \n\t"
00752
00753 #define MULADDC_STOP \
00754 "sw $12, %0 \n\t" \
00755 "sw $11, %1 \n\t" \
00756 "sw $10, %2 \n\t" \
00757 : "=m" (c), "=m" (d), "=m" (s) \
00758 : "m" (s), "m" (d), "m" (c), "m" (b) \
00759 : "$9", "$10", "$11", "$12", "$13", "$14", "$15", "lo", "hi" \
00760 );
00761
00762 #endif
00763 #endif
00764
00765 #if (defined(_MSC_VER) && defined(_M_IX86)) || defined(__WATCOMC__)
00766
00767 #define MULADDC_INIT \
00768 __asm mov esi, s \
00769 __asm mov edi, d \
00770 __asm mov ecx, c \
00771 __asm mov ebx, b
00772
00773 #define MULADDC_CORE \
00774 __asm lodsd \
00775 __asm mul ebx \
00776 __asm add eax, ecx \
00777 __asm adc edx, 0 \
00778 __asm add eax, [edi] \
00779 __asm adc edx, 0 \
00780 __asm mov ecx, edx \
00781 __asm stosd
00782
00783 #if defined(MBEDTLS_HAVE_SSE2)
00784
00785 #define EMIT __asm _emit
00786
00787 #define MULADDC_HUIT \
00788 EMIT 0x0F EMIT 0x6E EMIT 0xC9 \
00789 EMIT 0x0F EMIT 0x6E EMIT 0xC3 \
00790 EMIT 0x0F EMIT 0x6E EMIT 0x1F \
00791 EMIT 0x0F EMIT 0xD4 EMIT 0xCB \
00792 EMIT 0x0F EMIT 0x6E EMIT 0x16 \
00793 EMIT 0x0F EMIT 0xF4 EMIT 0xD0 \
00794 EMIT 0x0F EMIT 0x6E EMIT 0x66 EMIT 0x04 \
00795 EMIT 0x0F EMIT 0xF4 EMIT 0xE0 \
00796 EMIT 0x0F EMIT 0x6E EMIT 0x76 EMIT 0x08 \
00797 EMIT 0x0F EMIT 0xF4 EMIT 0xF0 \
00798 EMIT 0x0F EMIT 0x6E EMIT 0x7E EMIT 0x0C \
00799 EMIT 0x0F EMIT 0xF4 EMIT 0xF8 \
00800 EMIT 0x0F EMIT 0xD4 EMIT 0xCA \
00801 EMIT 0x0F EMIT 0x6E EMIT 0x5F EMIT 0x04 \
00802 EMIT 0x0F EMIT 0xD4 EMIT 0xDC \
00803 EMIT 0x0F EMIT 0x6E EMIT 0x6F EMIT 0x08 \
00804 EMIT 0x0F EMIT 0xD4 EMIT 0xEE \
00805 EMIT 0x0F EMIT 0x6E EMIT 0x67 EMIT 0x0C \
00806 EMIT 0x0F EMIT 0xD4 EMIT 0xFC \
00807 EMIT 0x0F EMIT 0x7E EMIT 0x0F \
00808 EMIT 0x0F EMIT 0x6E EMIT 0x56 EMIT 0x10 \
00809 EMIT 0x0F EMIT 0xF4 EMIT 0xD0 \
00810 EMIT 0x0F EMIT 0x73 EMIT 0xD1 EMIT 0x20 \
00811 EMIT 0x0F EMIT 0x6E EMIT 0x66 EMIT 0x14 \
00812 EMIT 0x0F EMIT 0xF4 EMIT 0xE0 \
00813 EMIT 0x0F EMIT 0xD4 EMIT 0xCB \
00814 EMIT 0x0F EMIT 0x6E EMIT 0x76 EMIT 0x18 \
00815 EMIT 0x0F EMIT 0xF4 EMIT 0xF0 \
00816 EMIT 0x0F EMIT 0x7E EMIT 0x4F EMIT 0x04 \
00817 EMIT 0x0F EMIT 0x73 EMIT 0xD1 EMIT 0x20 \
00818 EMIT 0x0F EMIT 0x6E EMIT 0x5E EMIT 0x1C \
00819 EMIT 0x0F EMIT 0xF4 EMIT 0xD8 \
00820 EMIT 0x0F EMIT 0xD4 EMIT 0xCD \
00821 EMIT 0x0F EMIT 0x6E EMIT 0x6F EMIT 0x10 \
00822 EMIT 0x0F EMIT 0xD4 EMIT 0xD5 \
00823 EMIT 0x0F EMIT 0x7E EMIT 0x4F EMIT 0x08 \
00824 EMIT 0x0F EMIT 0x73 EMIT 0xD1 EMIT 0x20 \
00825 EMIT 0x0F EMIT 0xD4 EMIT 0xCF \
00826 EMIT 0x0F EMIT 0x6E EMIT 0x6F EMIT 0x14 \
00827 EMIT 0x0F EMIT 0xD4 EMIT 0xE5 \
00828 EMIT 0x0F EMIT 0x7E EMIT 0x4F EMIT 0x0C \
00829 EMIT 0x0F EMIT 0x73 EMIT 0xD1 EMIT 0x20 \
00830 EMIT 0x0F EMIT 0xD4 EMIT 0xCA \
00831 EMIT 0x0F EMIT 0x6E EMIT 0x6F EMIT 0x18 \
00832 EMIT 0x0F EMIT 0xD4 EMIT 0xF5 \
00833 EMIT 0x0F EMIT 0x7E EMIT 0x4F EMIT 0x10 \
00834 EMIT 0x0F EMIT 0x73 EMIT 0xD1 EMIT 0x20 \
00835 EMIT 0x0F EMIT 0xD4 EMIT 0xCC \
00836 EMIT 0x0F EMIT 0x6E EMIT 0x6F EMIT 0x1C \
00837 EMIT 0x0F EMIT 0xD4 EMIT 0xDD \
00838 EMIT 0x0F EMIT 0x7E EMIT 0x4F EMIT 0x14 \
00839 EMIT 0x0F EMIT 0x73 EMIT 0xD1 EMIT 0x20 \
00840 EMIT 0x0F EMIT 0xD4 EMIT 0xCE \
00841 EMIT 0x0F EMIT 0x7E EMIT 0x4F EMIT 0x18 \
00842 EMIT 0x0F EMIT 0x73 EMIT 0xD1 EMIT 0x20 \
00843 EMIT 0x0F EMIT 0xD4 EMIT 0xCB \
00844 EMIT 0x0F EMIT 0x7E EMIT 0x4F EMIT 0x1C \
00845 EMIT 0x83 EMIT 0xC7 EMIT 0x20 \
00846 EMIT 0x83 EMIT 0xC6 EMIT 0x20 \
00847 EMIT 0x0F EMIT 0x73 EMIT 0xD1 EMIT 0x20 \
00848 EMIT 0x0F EMIT 0x7E EMIT 0xC9
00849
00850 #define MULADDC_STOP \
00851 EMIT 0x0F EMIT 0x77 \
00852 __asm mov c, ecx \
00853 __asm mov d, edi \
00854 __asm mov s, esi \
00855
00856 #else
00857
00858 #define MULADDC_STOP \
00859 __asm mov c, ecx \
00860 __asm mov d, edi \
00861 __asm mov s, esi \
00862
00863 #endif
00864 #endif
00865
00866 #endif
00867
00868 #if !defined(MULADDC_CORE)
00869 #if defined(MBEDTLS_HAVE_UDBL)
00870
00871 #define MULADDC_INIT \
00872 { \
00873 mbedtls_t_udbl r; \
00874 mbedtls_mpi_uint r0, r1;
00875
00876 #define MULADDC_CORE \
00877 r = *(s++) * (mbedtls_t_udbl) b; \
00878 r0 = (mbedtls_mpi_uint) r; \
00879 r1 = (mbedtls_mpi_uint)( r >> biL ); \
00880 r0 += c; r1 += (r0 < c); \
00881 r0 += *d; r1 += (r0 < *d); \
00882 c = r1; *(d++) = r0;
00883
00884 #define MULADDC_STOP \
00885 }
00886
00887 #else
00888 #define MULADDC_INIT \
00889 { \
00890 mbedtls_mpi_uint s0, s1, b0, b1; \
00891 mbedtls_mpi_uint r0, r1, rx, ry; \
00892 b0 = ( b << biH ) >> biH; \
00893 b1 = ( b >> biH );
00894
00895 #define MULADDC_CORE \
00896 s0 = ( *s << biH ) >> biH; \
00897 s1 = ( *s >> biH ); s++; \
00898 rx = s0 * b1; r0 = s0 * b0; \
00899 ry = s1 * b0; r1 = s1 * b1; \
00900 r1 += ( rx >> biH ); \
00901 r1 += ( ry >> biH ); \
00902 rx <<= biH; ry <<= biH; \
00903 r0 += rx; r1 += (r0 < rx); \
00904 r0 += ry; r1 += (r0 < ry); \
00905 r0 += c; r1 += (r0 < c); \
00906 r0 += *d; r1 += (r0 < *d); \
00907 c = r1; *(d++) = r0;
00908
00909 #define MULADDC_STOP \
00910 }
00911
00912 #endif
00913 #endif
00914
00915 #endif