Grid 0.7.0
WilsonKernelsHandImplementation.h
Go to the documentation of this file.
1 /*************************************************************************************
2
3 Grid physics library, www.github.com/paboyle/Grid
4
5 Source file: ./lib/qcd/action/fermion/WilsonKernelsHand.cc
6
7 Copyright (C) 2015
8
9Author: Peter Boyle <paboyle@ph.ed.ac.uk>
10Author: paboyle <paboyle@ph.ed.ac.uk>
11
12 This program is free software; you can redistribute it and/or modify
13 it under the terms of the GNU General Public License as published by
14 the Free Software Foundation; either version 2 of the License, or
15 (at your option) any later version.
16
17 This program is distributed in the hope that it will be useful,
18 but WITHOUT ANY WARRANTY; without even the implied warranty of
19 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
20 GNU General Public License for more details.
21
22 You should have received a copy of the GNU General Public License along
23 with this program; if not, write to the Free Software Foundation, Inc.,
24 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
25
26 See the full license in the file "LICENSE" in the top level distribution directory
27 *************************************************************************************/
28 /* END LEGAL */
29
30#pragma once
31
32#include <Grid/qcd/action/fermion/FermionCore.h>
33
34
35#undef LOAD_CHIMU
36#undef LOAD_CHI
37#undef MULT_2SPIN
38#undef PERMUTE_DIR
39#undef XP_PROJ
40#undef YP_PROJ
41#undef ZP_PROJ
42#undef TP_PROJ
43#undef XM_PROJ
44#undef YM_PROJ
45#undef ZM_PROJ
46#undef TM_PROJ
47#undef XP_RECON
48#undef XP_RECON_ACCUM
49#undef XM_RECON
50#undef XM_RECON_ACCUM
51#undef YP_RECON_ACCUM
52#undef YM_RECON_ACCUM
53#undef ZP_RECON_ACCUM
54#undef ZM_RECON_ACCUM
55#undef TP_RECON_ACCUM
56#undef TM_RECON_ACCUM
57#undef ZERO_RESULT
58#undef Chimu_00
59#undef Chimu_01
60#undef Chimu_02
61#undef Chimu_10
62#undef Chimu_11
63#undef Chimu_12
64#undef Chimu_20
65#undef Chimu_21
66#undef Chimu_22
67#undef Chimu_30
68#undef Chimu_31
69#undef Chimu_32
70#undef HAND_STENCIL_LEG
71#undef HAND_STENCIL_LEG_INT
72#undef HAND_STENCIL_LEG_EXT
73#undef HAND_RESULT
74#undef HAND_RESULT_INT
75#undef HAND_RESULT_EXT
76
77#define REGISTER
78
79#ifdef GRID_SIMT
80#define LOAD_CHIMU(Ptype) \
81 {const SiteSpinor & ref (in[offset]); \
82 Chimu_00=coalescedReadPermute<Ptype>(ref()(0)(0),perm,lane); \
83 Chimu_01=coalescedReadPermute<Ptype>(ref()(0)(1),perm,lane); \
84 Chimu_02=coalescedReadPermute<Ptype>(ref()(0)(2),perm,lane); \
85 Chimu_10=coalescedReadPermute<Ptype>(ref()(1)(0),perm,lane); \
86 Chimu_11=coalescedReadPermute<Ptype>(ref()(1)(1),perm,lane); \
87 Chimu_12=coalescedReadPermute<Ptype>(ref()(1)(2),perm,lane); \
88 Chimu_20=coalescedReadPermute<Ptype>(ref()(2)(0),perm,lane); \
89 Chimu_21=coalescedReadPermute<Ptype>(ref()(2)(1),perm,lane); \
90 Chimu_22=coalescedReadPermute<Ptype>(ref()(2)(2),perm,lane); \
91 Chimu_30=coalescedReadPermute<Ptype>(ref()(3)(0),perm,lane); \
92 Chimu_31=coalescedReadPermute<Ptype>(ref()(3)(1),perm,lane); \
93 Chimu_32=coalescedReadPermute<Ptype>(ref()(3)(2),perm,lane); }
94#define PERMUTE_DIR(dir) ;
95#else
96#define LOAD_CHIMU(Ptype) \
97 {const SiteSpinor & ref (in[offset]); \
98 Chimu_00=ref()(0)(0);\
99 Chimu_01=ref()(0)(1);\
100 Chimu_02=ref()(0)(2);\
101 Chimu_10=ref()(1)(0);\
102 Chimu_11=ref()(1)(1);\
103 Chimu_12=ref()(1)(2);\
104 Chimu_20=ref()(2)(0);\
105 Chimu_21=ref()(2)(1);\
106 Chimu_22=ref()(2)(2);\
107 Chimu_30=ref()(3)(0);\
108 Chimu_31=ref()(3)(1);\
109 Chimu_32=ref()(3)(2);}
110
111#define PERMUTE_DIR(dir) \
112 permute##dir(Chi_00,Chi_00); \
113 permute##dir(Chi_01,Chi_01); \
114 permute##dir(Chi_02,Chi_02); \
115 permute##dir(Chi_10,Chi_10); \
116 permute##dir(Chi_11,Chi_11); \
117 permute##dir(Chi_12,Chi_12);
118
119#endif
120
121#define MULT_2SPIN(A)\
122 {auto & ref(U[sU](A)); \
123 U_00=coalescedRead(ref()(0,0),lane); \
124 U_10=coalescedRead(ref()(1,0),lane); \
125 U_20=coalescedRead(ref()(2,0),lane); \
126 U_01=coalescedRead(ref()(0,1),lane); \
127 U_11=coalescedRead(ref()(1,1),lane); \
128 U_21=coalescedRead(ref()(2,1),lane); \
129 UChi_00 = U_00*Chi_00; \
130 UChi_10 = U_00*Chi_10; \
131 UChi_01 = U_10*Chi_00; \
132 UChi_11 = U_10*Chi_10; \
133 UChi_02 = U_20*Chi_00; \
134 UChi_12 = U_20*Chi_10; \
135 UChi_00+= U_01*Chi_01; \
136 UChi_10+= U_01*Chi_11; \
137 UChi_01+= U_11*Chi_01; \
138 UChi_11+= U_11*Chi_11; \
139 UChi_02+= U_21*Chi_01; \
140 UChi_12+= U_21*Chi_11; \
141 U_00=coalescedRead(ref()(0,2),lane); \
142 U_10=coalescedRead(ref()(1,2),lane); \
143 U_20=coalescedRead(ref()(2,2),lane); \
144 UChi_00+= U_00*Chi_02; \
145 UChi_10+= U_00*Chi_12; \
146 UChi_01+= U_10*Chi_02; \
147 UChi_11+= U_10*Chi_12; \
148 UChi_02+= U_20*Chi_02; \
149 UChi_12+= U_20*Chi_12;}
150
151#define LOAD_CHI \
152 {const SiteHalfSpinor &ref(buf[offset]); \
153 Chi_00 = coalescedRead(ref()(0)(0),lane); \
154 Chi_01 = coalescedRead(ref()(0)(1),lane); \
155 Chi_02 = coalescedRead(ref()(0)(2),lane); \
156 Chi_10 = coalescedRead(ref()(1)(0),lane); \
157 Chi_11 = coalescedRead(ref()(1)(1),lane); \
158 Chi_12 = coalescedRead(ref()(1)(2),lane);}
159
160// hspin(0)=fspin(0)+timesI(fspin(3));
161// hspin(1)=fspin(1)+timesI(fspin(2));
162#define XP_PROJ \
163 Chi_00 = Chimu_00+timesI(Chimu_30);\
164 Chi_01 = Chimu_01+timesI(Chimu_31);\
165 Chi_02 = Chimu_02+timesI(Chimu_32);\
166 Chi_10 = Chimu_10+timesI(Chimu_20);\
167 Chi_11 = Chimu_11+timesI(Chimu_21);\
168 Chi_12 = Chimu_12+timesI(Chimu_22);
169
170#define YP_PROJ \
171 Chi_00 = Chimu_00-Chimu_30;\
172 Chi_01 = Chimu_01-Chimu_31;\
173 Chi_02 = Chimu_02-Chimu_32;\
174 Chi_10 = Chimu_10+Chimu_20;\
175 Chi_11 = Chimu_11+Chimu_21;\
176 Chi_12 = Chimu_12+Chimu_22;
177
178#define ZP_PROJ \
179 Chi_00 = Chimu_00+timesI(Chimu_20); \
180 Chi_01 = Chimu_01+timesI(Chimu_21); \
181 Chi_02 = Chimu_02+timesI(Chimu_22); \
182 Chi_10 = Chimu_10-timesI(Chimu_30); \
183 Chi_11 = Chimu_11-timesI(Chimu_31); \
184 Chi_12 = Chimu_12-timesI(Chimu_32);
185
186#define TP_PROJ \
187 Chi_00 = Chimu_00+Chimu_20; \
188 Chi_01 = Chimu_01+Chimu_21; \
189 Chi_02 = Chimu_02+Chimu_22; \
190 Chi_10 = Chimu_10+Chimu_30; \
191 Chi_11 = Chimu_11+Chimu_31; \
192 Chi_12 = Chimu_12+Chimu_32;
193
194
195// hspin(0)=fspin(0)-timesI(fspin(3));
196// hspin(1)=fspin(1)-timesI(fspin(2));
197#define XM_PROJ \
198 Chi_00 = Chimu_00-timesI(Chimu_30);\
199 Chi_01 = Chimu_01-timesI(Chimu_31);\
200 Chi_02 = Chimu_02-timesI(Chimu_32);\
201 Chi_10 = Chimu_10-timesI(Chimu_20);\
202 Chi_11 = Chimu_11-timesI(Chimu_21);\
203 Chi_12 = Chimu_12-timesI(Chimu_22);
204
205#define YM_PROJ \
206 Chi_00 = Chimu_00+Chimu_30;\
207 Chi_01 = Chimu_01+Chimu_31;\
208 Chi_02 = Chimu_02+Chimu_32;\
209 Chi_10 = Chimu_10-Chimu_20;\
210 Chi_11 = Chimu_11-Chimu_21;\
211 Chi_12 = Chimu_12-Chimu_22;
212
213#define ZM_PROJ \
214 Chi_00 = Chimu_00-timesI(Chimu_20); \
215 Chi_01 = Chimu_01-timesI(Chimu_21); \
216 Chi_02 = Chimu_02-timesI(Chimu_22); \
217 Chi_10 = Chimu_10+timesI(Chimu_30); \
218 Chi_11 = Chimu_11+timesI(Chimu_31); \
219 Chi_12 = Chimu_12+timesI(Chimu_32);
220
221#define TM_PROJ \
222 Chi_00 = Chimu_00-Chimu_20; \
223 Chi_01 = Chimu_01-Chimu_21; \
224 Chi_02 = Chimu_02-Chimu_22; \
225 Chi_10 = Chimu_10-Chimu_30; \
226 Chi_11 = Chimu_11-Chimu_31; \
227 Chi_12 = Chimu_12-Chimu_32;
228
229// fspin(0)=hspin(0);
230// fspin(1)=hspin(1);
231// fspin(2)=timesMinusI(hspin(1));
232// fspin(3)=timesMinusI(hspin(0));
233#define XP_RECON\
234 result_00 = UChi_00;\
235 result_01 = UChi_01;\
236 result_02 = UChi_02;\
237 result_10 = UChi_10;\
238 result_11 = UChi_11;\
239 result_12 = UChi_12;\
240 result_20 = timesMinusI(UChi_10);\
241 result_21 = timesMinusI(UChi_11);\
242 result_22 = timesMinusI(UChi_12);\
243 result_30 = timesMinusI(UChi_00);\
244 result_31 = timesMinusI(UChi_01);\
245 result_32 = timesMinusI(UChi_02);
246
247#define XP_RECON_ACCUM\
248 result_00+=UChi_00;\
249 result_01+=UChi_01;\
250 result_02+=UChi_02;\
251 result_10+=UChi_10;\
252 result_11+=UChi_11;\
253 result_12+=UChi_12;\
254 result_20-=timesI(UChi_10);\
255 result_21-=timesI(UChi_11);\
256 result_22-=timesI(UChi_12);\
257 result_30-=timesI(UChi_00);\
258 result_31-=timesI(UChi_01);\
259 result_32-=timesI(UChi_02);
260
261#define XM_RECON\
262 result_00 = UChi_00;\
263 result_01 = UChi_01;\
264 result_02 = UChi_02;\
265 result_10 = UChi_10;\
266 result_11 = UChi_11;\
267 result_12 = UChi_12;\
268 result_20 = timesI(UChi_10);\
269 result_21 = timesI(UChi_11);\
270 result_22 = timesI(UChi_12);\
271 result_30 = timesI(UChi_00);\
272 result_31 = timesI(UChi_01);\
273 result_32 = timesI(UChi_02);
274
275#define XM_RECON_ACCUM\
276 result_00+= UChi_00;\
277 result_01+= UChi_01;\
278 result_02+= UChi_02;\
279 result_10+= UChi_10;\
280 result_11+= UChi_11;\
281 result_12+= UChi_12;\
282 result_20+= timesI(UChi_10);\
283 result_21+= timesI(UChi_11);\
284 result_22+= timesI(UChi_12);\
285 result_30+= timesI(UChi_00);\
286 result_31+= timesI(UChi_01);\
287 result_32+= timesI(UChi_02);
288
289#define YP_RECON_ACCUM\
290 result_00+= UChi_00;\
291 result_01+= UChi_01;\
292 result_02+= UChi_02;\
293 result_10+= UChi_10;\
294 result_11+= UChi_11;\
295 result_12+= UChi_12;\
296 result_20+= UChi_10;\
297 result_21+= UChi_11;\
298 result_22+= UChi_12;\
299 result_30-= UChi_00;\
300 result_31-= UChi_01;\
301 result_32-= UChi_02;
302
303#define YM_RECON_ACCUM\
304 result_00+= UChi_00;\
305 result_01+= UChi_01;\
306 result_02+= UChi_02;\
307 result_10+= UChi_10;\
308 result_11+= UChi_11;\
309 result_12+= UChi_12;\
310 result_20-= UChi_10;\
311 result_21-= UChi_11;\
312 result_22-= UChi_12;\
313 result_30+= UChi_00;\
314 result_31+= UChi_01;\
315 result_32+= UChi_02;
316
317#define ZP_RECON_ACCUM\
318 result_00+= UChi_00;\
319 result_01+= UChi_01;\
320 result_02+= UChi_02;\
321 result_10+= UChi_10;\
322 result_11+= UChi_11;\
323 result_12+= UChi_12;\
324 result_20-= timesI(UChi_00); \
325 result_21-= timesI(UChi_01); \
326 result_22-= timesI(UChi_02); \
327 result_30+= timesI(UChi_10); \
328 result_31+= timesI(UChi_11); \
329 result_32+= timesI(UChi_12);
330
331#define ZM_RECON_ACCUM\
332 result_00+= UChi_00;\
333 result_01+= UChi_01;\
334 result_02+= UChi_02;\
335 result_10+= UChi_10;\
336 result_11+= UChi_11;\
337 result_12+= UChi_12;\
338 result_20+= timesI(UChi_00); \
339 result_21+= timesI(UChi_01); \
340 result_22+= timesI(UChi_02); \
341 result_30-= timesI(UChi_10); \
342 result_31-= timesI(UChi_11); \
343 result_32-= timesI(UChi_12);
344
345#define TP_RECON_ACCUM\
346 result_00+= UChi_00;\
347 result_01+= UChi_01;\
348 result_02+= UChi_02;\
349 result_10+= UChi_10;\
350 result_11+= UChi_11;\
351 result_12+= UChi_12;\
352 result_20+= UChi_00; \
353 result_21+= UChi_01; \
354 result_22+= UChi_02; \
355 result_30+= UChi_10; \
356 result_31+= UChi_11; \
357 result_32+= UChi_12;
358
359#define TM_RECON_ACCUM\
360 result_00+= UChi_00;\
361 result_01+= UChi_01;\
362 result_02+= UChi_02;\
363 result_10+= UChi_10;\
364 result_11+= UChi_11;\
365 result_12+= UChi_12;\
366 result_20-= UChi_00; \
367 result_21-= UChi_01; \
368 result_22-= UChi_02; \
369 result_30-= UChi_10; \
370 result_31-= UChi_11; \
371 result_32-= UChi_12;
372
373#define HAND_STENCIL_LEGB(PROJ,PERM,DIR,RECON) \
374 {int ptype; \
375 SE=st.GetEntry(ptype,DIR,ss); \
376 auto offset = SE->_offset; \
377 auto local = SE->_is_local; \
378 auto perm = SE->_permute; \
379 if ( local ) { \
380 LOAD_CHIMU(PERM); \
381 PROJ; \
382 if ( perm) { \
383 PERMUTE_DIR(PERM); \
384 } \
385 } else { \
386 LOAD_CHI; \
387 } \
388 acceleratorSynchronise(); \
389 MULT_2SPIN(DIR); \
390 RECON; }
391
392#define HAND_STENCIL_LEG(PROJ,PERM,DIR,RECON) \
393 { SE=&st_p[DIR+8*ss]; \
394 auto ptype=st_perm[DIR]; \
395 auto offset = SE->_offset; \
396 auto local = SE->_is_local; \
397 auto perm = SE->_permute; \
398 if ( local ) { \
399 LOAD_CHIMU(PERM); \
400 PROJ; \
401 if ( perm) { \
402 PERMUTE_DIR(PERM); \
403 } \
404 } else { \
405 LOAD_CHI; \
406 } \
407 acceleratorSynchronise(); \
408 MULT_2SPIN(DIR); \
409 RECON; }
410
411#define HAND_STENCIL_LEGA(PROJ,PERM,DIR,RECON) \
412 { SE=&st_p[DIR+8*ss]; \
413 auto ptype=st_perm[DIR]; \
414 /*SE=st.GetEntry(ptype,DIR,ss);*/ \
415 auto offset = SE->_offset; \
416 auto perm = SE->_permute; \
417 LOAD_CHIMU(PERM); \
418 PROJ; \
419 MULT_2SPIN(DIR); \
420 RECON; }
421
422#define HAND_STENCIL_LEG_INT(PROJ,PERM,DIR,RECON) \
423 { int ptype; \
424 SE=st.GetEntry(ptype,DIR,ss); \
425 auto offset = SE->_offset; \
426 auto local = SE->_is_local; \
427 auto perm = SE->_permute; \
428 if ( local ) { \
429 LOAD_CHIMU(PERM); \
430 PROJ; \
431 if ( perm) { \
432 PERMUTE_DIR(PERM); \
433 } \
434 } else if ( st.same_node[DIR] ) { \
435 LOAD_CHI; \
436 } \
437 acceleratorSynchronise(); \
438 if (local || st.same_node[DIR] ) { \
439 MULT_2SPIN(DIR); \
440 RECON; \
441 } \
442 acceleratorSynchronise(); }
443
444#define HAND_STENCIL_LEG_EXT(PROJ,PERM,DIR,RECON) \
445 { int ptype; \
446 SE=st.GetEntry(ptype,DIR,ss); \
447 auto offset = SE->_offset; \
448 if((!SE->_is_local)&&(!st.same_node[DIR]) ) { \
449 LOAD_CHI; \
450 MULT_2SPIN(DIR); \
451 RECON; \
452 nmu++; \
453 } \
454 acceleratorSynchronise(); }
455
456#define HAND_RESULT(ss) \
457 { \
458 SiteSpinor & ref (out[ss]); \
459 coalescedWrite(ref()(0)(0),result_00,lane); \
460 coalescedWrite(ref()(0)(1),result_01,lane); \
461 coalescedWrite(ref()(0)(2),result_02,lane); \
462 coalescedWrite(ref()(1)(0),result_10,lane); \
463 coalescedWrite(ref()(1)(1),result_11,lane); \
464 coalescedWrite(ref()(1)(2),result_12,lane); \
465 coalescedWrite(ref()(2)(0),result_20,lane); \
466 coalescedWrite(ref()(2)(1),result_21,lane); \
467 coalescedWrite(ref()(2)(2),result_22,lane); \
468 coalescedWrite(ref()(3)(0),result_30,lane); \
469 coalescedWrite(ref()(3)(1),result_31,lane); \
470 coalescedWrite(ref()(3)(2),result_32,lane); \
471 }
472
473#define HAND_RESULT_EXT(ss) \
474 { \
475 SiteSpinor & ref (out[ss]); \
476 coalescedWrite(ref()(0)(0),coalescedRead(ref()(0)(0))+result_00,lane); \
477 coalescedWrite(ref()(0)(1),coalescedRead(ref()(0)(1))+result_01,lane); \
478 coalescedWrite(ref()(0)(2),coalescedRead(ref()(0)(2))+result_02,lane); \
479 coalescedWrite(ref()(1)(0),coalescedRead(ref()(1)(0))+result_10,lane); \
480 coalescedWrite(ref()(1)(1),coalescedRead(ref()(1)(1))+result_11,lane); \
481 coalescedWrite(ref()(1)(2),coalescedRead(ref()(1)(2))+result_12,lane); \
482 coalescedWrite(ref()(2)(0),coalescedRead(ref()(2)(0))+result_20,lane); \
483 coalescedWrite(ref()(2)(1),coalescedRead(ref()(2)(1))+result_21,lane); \
484 coalescedWrite(ref()(2)(2),coalescedRead(ref()(2)(2))+result_22,lane); \
485 coalescedWrite(ref()(3)(0),coalescedRead(ref()(3)(0))+result_30,lane); \
486 coalescedWrite(ref()(3)(1),coalescedRead(ref()(3)(1))+result_31,lane); \
487 coalescedWrite(ref()(3)(2),coalescedRead(ref()(3)(2))+result_32,lane); \
488 }
489
490#define HAND_DECLARATIONS(Simd) \
491 Simd result_00; \
492 Simd result_01; \
493 Simd result_02; \
494 Simd result_10; \
495 Simd result_11; \
496 Simd result_12; \
497 Simd result_20; \
498 Simd result_21; \
499 Simd result_22; \
500 Simd result_30; \
501 Simd result_31; \
502 Simd result_32; \
503 Simd Chi_00; \
504 Simd Chi_01; \
505 Simd Chi_02; \
506 Simd Chi_10; \
507 Simd Chi_11; \
508 Simd Chi_12; \
509 Simd UChi_00; \
510 Simd UChi_01; \
511 Simd UChi_02; \
512 Simd UChi_10; \
513 Simd UChi_11; \
514 Simd UChi_12; \
515 Simd U_00; \
516 Simd U_10; \
517 Simd U_20; \
518 Simd U_01; \
519 Simd U_11; \
520 Simd U_21;
521
522#define ZERO_RESULT \
523 zeroit(result_00); \
524 zeroit(result_01); \
525 zeroit(result_02); \
526 zeroit(result_10); \
527 zeroit(result_11); \
528 zeroit(result_12); \
529 zeroit(result_20); \
530 zeroit(result_21); \
531 zeroit(result_22); \
532 zeroit(result_30); \
533 zeroit(result_31); \
534 zeroit(result_32);
535
536#define Chimu_00 Chi_00
537#define Chimu_01 Chi_01
538#define Chimu_02 Chi_02
539#define Chimu_10 Chi_10
540#define Chimu_11 Chi_11
541#define Chimu_12 Chi_12
542#define Chimu_20 UChi_00
543#define Chimu_21 UChi_01
544#define Chimu_22 UChi_02
545#define Chimu_30 UChi_10
546#define Chimu_31 UChi_11
547#define Chimu_32 UChi_12
548
549NAMESPACE_BEGIN(Grid);
550
551
552#ifdef SYCL_HACK
553template<class Impl> accelerator_inline void
554WilsonKernels<Impl>::HandDhopSiteSycl(StencilVector st_perm,StencilEntry *st_p, SiteDoubledGaugeField *U,SiteHalfSpinor *buf,
555 int ss,int sU,const SiteSpinor *in, SiteSpinor *out)
556{
557// T==0, Z==1, Y==2, Z==3 expect 1,2,2,2 simd layout etc...
558 typedef typename Simd::scalar_type S;
559 typedef typename Simd::vector_type V;
560 typedef iSinglet<Simd> vCplx;
561 // typedef decltype( coalescedRead( vCplx()()() )) Simt;
562 typedef decltype( coalescedRead( in[0]()(0)(0) )) Simt;
563
564 const int Nsimd = SiteHalfSpinor::Nsimd();
565 const int lane=acceleratorSIMTlane(Nsimd);
566
567 HAND_DECLARATIONS(Simt);
568
569 StencilEntry *SE;
570 HAND_STENCIL_LEG(XM_PROJ,3,Xp,XM_RECON);
571 HAND_STENCIL_LEG(YM_PROJ,2,Yp,YM_RECON_ACCUM);
572 HAND_STENCIL_LEG(ZM_PROJ,1,Zp,ZM_RECON_ACCUM);
573 HAND_STENCIL_LEG(TM_PROJ,0,Tp,TM_RECON_ACCUM);
574 HAND_STENCIL_LEG(XP_PROJ,3,Xm,XP_RECON_ACCUM);
575 HAND_STENCIL_LEG(YP_PROJ,2,Ym,YP_RECON_ACCUM);
576 HAND_STENCIL_LEG(ZP_PROJ,1,Zm,ZP_RECON_ACCUM);
577 HAND_STENCIL_LEG(TP_PROJ,0,Tm,TP_RECON_ACCUM);
578 HAND_RESULT(ss);
579}
580#endif
581
582template<class Impl> accelerator_inline void
583WilsonKernels<Impl>::HandDhopSite(StencilView &st, DoubledGaugeFieldView &U,SiteHalfSpinor *buf,
584 int ss,int sU,const FermionFieldView &in, FermionFieldView &out)
585{
586 auto st_p = st._entries_p;
587 auto st_perm = st._permute_type;
588// T==0, Z==1, Y==2, Z==3 expect 1,2,2,2 simd layout etc...
589 typedef typename Simd::scalar_type S;
590 typedef typename Simd::vector_type V;
591 typedef decltype( coalescedRead( in[0]()(0)(0) )) Simt;
592
593 const int Nsimd = SiteHalfSpinor::Nsimd();
594 const int lane=acceleratorSIMTlane(Nsimd);
595
596 HAND_DECLARATIONS(Simt);
597
598 StencilEntry *SE;
599 HAND_STENCIL_LEG(XM_PROJ,3,Xp,XM_RECON);
600 HAND_STENCIL_LEG(YM_PROJ,2,Yp,YM_RECON_ACCUM);
601 HAND_STENCIL_LEG(ZM_PROJ,1,Zp,ZM_RECON_ACCUM);
602 HAND_STENCIL_LEG(TM_PROJ,0,Tp,TM_RECON_ACCUM);
603 HAND_STENCIL_LEG(XP_PROJ,3,Xm,XP_RECON_ACCUM);
604 HAND_STENCIL_LEG(YP_PROJ,2,Ym,YP_RECON_ACCUM);
605 HAND_STENCIL_LEG(ZP_PROJ,1,Zm,ZP_RECON_ACCUM);
606 HAND_STENCIL_LEG(TP_PROJ,0,Tm,TP_RECON_ACCUM);
607 HAND_RESULT(ss);
608}
609
610template<class Impl> accelerator_inline
611void WilsonKernels<Impl>::HandDhopSiteDag(StencilView &st,DoubledGaugeFieldView &U,SiteHalfSpinor *buf,
612 int ss,int sU,const FermionFieldView &in, FermionFieldView &out)
613{
614 auto st_p = st._entries_p;
615 auto st_perm = st._permute_type;
616 typedef typename Simd::scalar_type S;
617 typedef typename Simd::vector_type V;
618 typedef decltype( coalescedRead( in[0]()(0)(0) )) Simt;
619
620 const int Nsimd = SiteHalfSpinor::Nsimd();
621 const int lane=acceleratorSIMTlane(Nsimd);
622
623 HAND_DECLARATIONS(Simt);
624
625 StencilEntry *SE;
626 HAND_STENCIL_LEG(XP_PROJ,3,Xp,XP_RECON);
627 HAND_STENCIL_LEG(YP_PROJ,2,Yp,YP_RECON_ACCUM);
628 HAND_STENCIL_LEG(ZP_PROJ,1,Zp,ZP_RECON_ACCUM);
629 HAND_STENCIL_LEG(TP_PROJ,0,Tp,TP_RECON_ACCUM);
630 HAND_STENCIL_LEG(XM_PROJ,3,Xm,XM_RECON_ACCUM);
631 HAND_STENCIL_LEG(YM_PROJ,2,Ym,YM_RECON_ACCUM);
632 HAND_STENCIL_LEG(ZM_PROJ,1,Zm,ZM_RECON_ACCUM);
633 HAND_STENCIL_LEG(TM_PROJ,0,Tm,TM_RECON_ACCUM);
634 HAND_RESULT(ss);
635}
636
637template<class Impl> accelerator_inline void
638WilsonKernels<Impl>::HandDhopSiteInt(StencilView &st,DoubledGaugeFieldView &U,SiteHalfSpinor *buf,
639 int ss,int sU,const FermionFieldView &in, FermionFieldView &out)
640{
641 // auto st_p = st._entries_p;
642 // auto st_perm = st._permute_type;
643// T==0, Z==1, Y==2, Z==3 expect 1,2,2,2 simd layout etc...
644 typedef typename Simd::scalar_type S;
645 typedef typename Simd::vector_type V;
646 typedef decltype( coalescedRead( in[0]()(0)(0) )) Simt;
647
648 const int Nsimd = SiteHalfSpinor::Nsimd();
649 const int lane=acceleratorSIMTlane(Nsimd);
650
651 HAND_DECLARATIONS(Simt);
652
653 StencilEntry *SE;
654 ZERO_RESULT;
655 HAND_STENCIL_LEG_INT(XM_PROJ,3,Xp,XM_RECON_ACCUM);
656 HAND_STENCIL_LEG_INT(YM_PROJ,2,Yp,YM_RECON_ACCUM);
657 HAND_STENCIL_LEG_INT(ZM_PROJ,1,Zp,ZM_RECON_ACCUM);
658 HAND_STENCIL_LEG_INT(TM_PROJ,0,Tp,TM_RECON_ACCUM);
659 HAND_STENCIL_LEG_INT(XP_PROJ,3,Xm,XP_RECON_ACCUM);
660 HAND_STENCIL_LEG_INT(YP_PROJ,2,Ym,YP_RECON_ACCUM);
661 HAND_STENCIL_LEG_INT(ZP_PROJ,1,Zm,ZP_RECON_ACCUM);
662 HAND_STENCIL_LEG_INT(TP_PROJ,0,Tm,TP_RECON_ACCUM);
663 HAND_RESULT(ss);
664}
665
666template<class Impl> accelerator_inline
667void WilsonKernels<Impl>::HandDhopSiteDagInt(StencilView &st,DoubledGaugeFieldView &U,SiteHalfSpinor *buf,
668 int ss,int sU,const FermionFieldView &in, FermionFieldView &out)
669{
670 // auto st_p = st._entries_p;
671 // auto st_perm = st._permute_type;
672 typedef typename Simd::scalar_type S;
673 typedef typename Simd::vector_type V;
674 typedef decltype( coalescedRead( in[0]()(0)(0) )) Simt;
675
676 const int Nsimd = SiteHalfSpinor::Nsimd();
677 const int lane=acceleratorSIMTlane(Nsimd);
678
679 HAND_DECLARATIONS(Simt);
680
681 StencilEntry *SE;
682 ZERO_RESULT;
683 HAND_STENCIL_LEG_INT(XP_PROJ,3,Xp,XP_RECON_ACCUM);
684 HAND_STENCIL_LEG_INT(YP_PROJ,2,Yp,YP_RECON_ACCUM);
685 HAND_STENCIL_LEG_INT(ZP_PROJ,1,Zp,ZP_RECON_ACCUM);
686 HAND_STENCIL_LEG_INT(TP_PROJ,0,Tp,TP_RECON_ACCUM);
687 HAND_STENCIL_LEG_INT(XM_PROJ,3,Xm,XM_RECON_ACCUM);
688 HAND_STENCIL_LEG_INT(YM_PROJ,2,Ym,YM_RECON_ACCUM);
689 HAND_STENCIL_LEG_INT(ZM_PROJ,1,Zm,ZM_RECON_ACCUM);
690 HAND_STENCIL_LEG_INT(TM_PROJ,0,Tm,TM_RECON_ACCUM);
691 HAND_RESULT(ss);
692}
693
694template<class Impl> accelerator_inline void
695WilsonKernels<Impl>::HandDhopSiteExt(StencilView &st,DoubledGaugeFieldView &U,SiteHalfSpinor *buf,
696 int ss,int sU,const FermionFieldView &in, FermionFieldView &out)
697{
698 // auto st_p = st._entries_p;
699 // auto st_perm = st._permute_type;
700// T==0, Z==1, Y==2, Z==3 expect 1,2,2,2 simd layout etc...
701 typedef typename Simd::scalar_type S;
702 typedef typename Simd::vector_type V;
703 typedef decltype( coalescedRead( in[0]()(0)(0) )) Simt;
704
705 const int Nsimd = SiteHalfSpinor::Nsimd();
706 const int lane=acceleratorSIMTlane(Nsimd);
707
708 HAND_DECLARATIONS(Simt);
709
710 // int offset, ptype;
711 StencilEntry *SE;
712 int nmu=0;
713 ZERO_RESULT;
714 HAND_STENCIL_LEG_EXT(XM_PROJ,3,Xp,XM_RECON_ACCUM);
715 HAND_STENCIL_LEG_EXT(YM_PROJ,2,Yp,YM_RECON_ACCUM);
716 HAND_STENCIL_LEG_EXT(ZM_PROJ,1,Zp,ZM_RECON_ACCUM);
717 HAND_STENCIL_LEG_EXT(TM_PROJ,0,Tp,TM_RECON_ACCUM);
718 HAND_STENCIL_LEG_EXT(XP_PROJ,3,Xm,XP_RECON_ACCUM);
719 HAND_STENCIL_LEG_EXT(YP_PROJ,2,Ym,YP_RECON_ACCUM);
720 HAND_STENCIL_LEG_EXT(ZP_PROJ,1,Zm,ZP_RECON_ACCUM);
721 HAND_STENCIL_LEG_EXT(TP_PROJ,0,Tm,TP_RECON_ACCUM);
722 HAND_RESULT_EXT(ss);
723}
724
725template<class Impl> accelerator_inline
726void WilsonKernels<Impl>::HandDhopSiteDagExt(StencilView &st,DoubledGaugeFieldView &U,SiteHalfSpinor *buf,
727 int ss,int sU,const FermionFieldView &in, FermionFieldView &out)
728{
729 // auto st_p = st._entries_p;
730 // auto st_perm = st._permute_type;
731 typedef typename Simd::scalar_type S;
732 typedef typename Simd::vector_type V;
733 typedef decltype( coalescedRead( in[0]()(0)(0) )) Simt;
734
735 const int Nsimd = SiteHalfSpinor::Nsimd();
736 const int lane=acceleratorSIMTlane(Nsimd);
737
738 HAND_DECLARATIONS(Simt);
739
740 StencilEntry *SE;
741 // int offset, ptype;
742 int nmu=0;
743 ZERO_RESULT;
744 HAND_STENCIL_LEG_EXT(XP_PROJ,3,Xp,XP_RECON_ACCUM);
745 HAND_STENCIL_LEG_EXT(YP_PROJ,2,Yp,YP_RECON_ACCUM);
746 HAND_STENCIL_LEG_EXT(ZP_PROJ,1,Zp,ZP_RECON_ACCUM);
747 HAND_STENCIL_LEG_EXT(TP_PROJ,0,Tp,TP_RECON_ACCUM);
748 HAND_STENCIL_LEG_EXT(XM_PROJ,3,Xm,XM_RECON_ACCUM);
749 HAND_STENCIL_LEG_EXT(YM_PROJ,2,Ym,YM_RECON_ACCUM);
750 HAND_STENCIL_LEG_EXT(ZM_PROJ,1,Zm,ZM_RECON_ACCUM);
751 HAND_STENCIL_LEG_EXT(TM_PROJ,0,Tm,TM_RECON_ACCUM);
752 HAND_RESULT_EXT(ss);
753}
754
756
757NAMESPACE_END(Grid);
758#undef LOAD_CHIMU
759#undef LOAD_CHI
760#undef MULT_2SPIN
761#undef PERMUTE_DIR
762#undef XP_PROJ
763#undef YP_PROJ
764#undef ZP_PROJ
765#undef TP_PROJ
766#undef XM_PROJ
767#undef YM_PROJ
768#undef ZM_PROJ
769#undef TM_PROJ
770#undef XP_RECON
771#undef XP_RECON_ACCUM
772#undef XM_RECON
773#undef XM_RECON_ACCUM
774#undef YP_RECON_ACCUM
775#undef YM_RECON_ACCUM
776#undef ZP_RECON_ACCUM
777#undef ZM_RECON_ACCUM
778#undef TP_RECON_ACCUM
779#undef TM_RECON_ACCUM
780#undef ZERO_RESULT
781#undef Chimu_00
782#undef Chimu_01
783#undef Chimu_02
784#undef Chimu_10
785#undef Chimu_11
786#undef Chimu_12
787#undef Chimu_20
788#undef Chimu_21
789#undef Chimu_22
790#undef Chimu_30
791#undef Chimu_31
792#undef Chimu_32
793#undef HAND_STENCIL_LEG
794#undef HAND_STENCIL_LEG_INT
795#undef HAND_STENCIL_LEG_EXT
796#undef HAND_RESULT
797#undef HAND_RESULT_INT
798#undef HAND_RESULT_EXT
799#undef HAND_DECLARATIONS
acceleratorSIMTlane
accelerator_inline int acceleratorSIMTlane(int Nsimd)
Definition Accelerator.h:614
accelerator_inline
#define accelerator_inline
Definition Accelerator.h:608
NAMESPACE_BEGIN
#define NAMESPACE_BEGIN(A)
Definition Namespace.h:35
NAMESPACE_END
#define NAMESPACE_END(A)
Definition Namespace.h:36
Xm
static constexpr int Xm
Definition QCD.h:45
Tm
static constexpr int Tm
Definition QCD.h:48
Tp
static constexpr int Tp
Definition QCD.h:44
Zp
static constexpr int Zp
Definition QCD.h:43
iSinglet
iScalar< iScalar< iScalar< vtype > > > iSinglet
Definition QCD.h:102
Zm
static constexpr int Zm
Definition QCD.h:47
Xp
static constexpr int Xp
Definition QCD.h:41
Yp
static constexpr int Yp
Definition QCD.h:42
Ym
static constexpr int Ym
Definition QCD.h:46
coalescedRead
accelerator_inline vobj coalescedRead(const vobj &__restrict__ vec, int lane=0)
Definition Tensor_SIMT.h:61
HAND_RESULT_EXT
#define HAND_RESULT_EXT(ss)
Definition WilsonKernelsHandImplementation.h:473
HAND_STENCIL_LEG_EXT
#define HAND_STENCIL_LEG_EXT(PROJ, PERM, DIR, RECON)
Definition WilsonKernelsHandImplementation.h:444
HAND_STENCIL_LEG
#define HAND_STENCIL_LEG(PROJ, PERM, DIR, RECON)
Definition WilsonKernelsHandImplementation.h:392
HAND_STENCIL_LEG_INT
#define HAND_STENCIL_LEG_INT(PROJ, PERM, DIR, RECON)
Definition WilsonKernelsHandImplementation.h:422
HAND_RESULT
#define HAND_RESULT(ss)
Definition WilsonKernelsHandImplementation.h:456
HAND_DECLARATIONS
#define HAND_DECLARATIONS(Simd)
Definition WilsonKernelsHandImplementation.h:490
U
static INTERNAL_PRECISION U
Definition Zolotarev.cc:230
WilsonKernels::HandDhopSiteExt
static accelerator void HandDhopSiteExt(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf, int sF, int sU, const FermionFieldView &in, FermionFieldView &out)
Definition WilsonKernelsHandImplementation.h:695
WilsonKernels::HandDhopSite
static accelerator void HandDhopSite(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf, int sF, int sU, const FermionFieldView &in, FermionFieldView &out)
Definition WilsonKernelsHandImplementation.h:583
WilsonKernels::HandDhopSiteDagExt
static accelerator void HandDhopSiteDagExt(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf, int sF, int sU, const FermionFieldView &in, FermionFieldView &out)
Definition WilsonKernelsHandImplementation.h:726
WilsonKernels::HandDhopSiteInt
static accelerator void HandDhopSiteInt(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf, int sF, int sU, const FermionFieldView &in, FermionFieldView &out)
Definition WilsonKernelsHandImplementation.h:638
WilsonKernels::HandDhopSiteDagInt
static accelerator void HandDhopSiteDagInt(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf, int sF, int sU, const FermionFieldView &in, FermionFieldView &out)
Definition WilsonKernelsHandImplementation.h:667
WilsonKernels::HandDhopSiteDag
static accelerator void HandDhopSiteDag(StencilView &st, DoubledGaugeFieldView &U, SiteHalfSpinor *buf, int sF, int sU, const FermionFieldView &in, FermionFieldView &out)
Definition WilsonKernelsHandImplementation.h:611