From 12fb96dcb4c26fdd2cb6bc00faeab9280d817fc9 Mon Sep 17 00:00:00 2001 From: LarissaBrencher Date: Fri, 28 Jan 2022 11:17:25 +0100 Subject: [PATCH 1/4] [test][parallel] Add partioning test with scotch --- test/CMakeLists.txt | 2 + test/parallel/CMakeLists.txt | 1 + test/parallel/partition/CMakeLists.txt | 6 + test/parallel/partition/ball.msh | 1173 ++++++++++++++++++++++++ test/parallel/partition/main.cc | 193 ++++ test/parallel/partition/rectangle.msh | 53 ++ 6 files changed, 1428 insertions(+) create mode 100644 test/parallel/CMakeLists.txt create mode 100644 test/parallel/partition/CMakeLists.txt create mode 100644 test/parallel/partition/ball.msh create mode 100644 test/parallel/partition/main.cc create mode 100644 test/parallel/partition/rectangle.msh diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 091a9b5089..1ac976185f 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -8,11 +8,13 @@ add_subdirectory(linear) add_subdirectory(material) add_subdirectory(multidomain) add_subdirectory(nonlinear) +add_subdirectory(parallel) add_subdirectory(porenetwork) add_subdirectory(porousmediumflow) add_subdirectory(discretization) add_subdirectory(timestepping) + # if Python bindings are enabled, include Python binding tests if(DUNE_ENABLE_PYTHONBINDINGS) if(${dune-common_VERSION} VERSION_GREATER_EQUAL 2.8) diff --git a/test/parallel/CMakeLists.txt b/test/parallel/CMakeLists.txt new file mode 100644 index 0000000000..3a976ee9a9 --- /dev/null +++ b/test/parallel/CMakeLists.txt @@ -0,0 +1 @@ +add_subdirectory(partition) diff --git a/test/parallel/partition/CMakeLists.txt b/test/parallel/partition/CMakeLists.txt new file mode 100644 index 0000000000..906451c366 --- /dev/null +++ b/test/parallel/partition/CMakeLists.txt @@ -0,0 +1,6 @@ +dune_symlink_to_source_files(FILES ball.msh rectangle.msh) +dumux_add_test(NAME test_parallel_partition_scotch + SOURCES main.cc + CMAKE_GUARD "( dune-uggrid_FOUND AND PTSCOTCH_FOUND )" + MPI_RANKS 1 2 3 4 + TIMEOUT 300) diff --git a/test/parallel/partition/ball.msh b/test/parallel/partition/ball.msh new file mode 100644 index 0000000000..4411e70a12 --- /dev/null +++ b/test/parallel/partition/ball.msh @@ -0,0 +1,1173 @@ +$MeshFormat +2.2 0 8 +$EndMeshFormat +$Nodes +198 +1 3.061616997868383e-17 -7.498798913309288e-33 0.5 +2 3.061616997868383e-17 -7.498798913309288e-33 -0.5 +3 0.1545084971874741 -3.784366730434159e-17 -0.4755282581475767 +4 0.2938926261462368 -7.198293278059974e-17 -0.4045084971874735 +5 0.4045084971874742 -9.907600726170927e-17 -0.2938926261462359 +6 0.475528258147577 -1.164708318489093e-16 -0.154508497187473 +7 0.5 -1.224646799147353e-16 7.657137397853899e-16 +8 0.4755282581475767 -1.164708318489092e-16 0.154508497187474 +9 0.4045084971874738 -9.907600726170918e-17 0.2938926261462365 +10 0.2938926261462371 -7.198293278059979e-17 0.4045084971874734 +11 0.1545084971874747 -3.784366730434174e-17 0.4755282581475765 +12 0.21857645514496 0.1367164652581294 0.4284074478619461 +13 0.4759451877014457 -0.132880263668928 0.07628246083544885 +14 0.2185763491207971 -0.1349086139510939 -0.428980239040003 +15 0.2286034258204459 -0.1400380918348344 0.4220542696602113 +16 0.4371044470969576 0.1517934865696115 0.1894688358625349 +17 0.4289802390399401 0.1349086139508836 -0.2185763491210502 +18 0.4289802390398977 -0.1349086139510624 -0.2185763491210233 +19 0.2185763491210322 0.1349086139510526 -0.4289802390398962 +20 0.3404406452539164 0.1349086139510439 0.3404406452539197 +21 0.4766150301258431 -0.1319242414550947 -0.07371639963155813 +22 0.4076225471962073 -0.2595209948849996 -0.1284239550519131 +23 0.3384368182696365 -0.2520431390962445 -0.2682065921524603 +24 0.3409896318710846 -0.1167465244715157 -0.3465491595434102 +25 0.07531627702520503 -0.1349086139510468 0.4755282581475784 +26 0.0753162770252058 0.134908613951052 -0.4755282581475768 +27 0.3370518872527516 0.1315243462512498 -0.3451048704995915 +28 0.3423072298998689 0.2597493212919576 -0.2556482944332813 +29 0.2353028861906338 0.2577589387604507 0.3580403346544128 +30 0.09305531851211013 0.241091096498632 0.4280371372739719 +31 0.4148873488943517 0.2532333456828638 -0.1172235486652734 +32 0.309219636002729 0.365205559163796 -0.1449417685355013 +33 0.1163827304876214 0.3573359627036667 0.3297970130290591 +34 0.2202955545038332 0.3560441472876059 -0.2733174598303047 +35 0.1739247379617718 0.4434073812114806 -0.1521186372937441 +36 0.07574391382782668 0.4158368425794604 -0.2671003179923001 +37 0.02249334671841327 0.4800410551776555 -0.1380385261342948 +38 0.2480652664013316 0.3580159470115432 0.2455365660969142 +39 0.1381633318218392 0.4304795613356104 0.2135374463933406 +40 0.3605520145873967 0.3464059421213085 -0.002273332359847931 +41 0.1230939452817091 0.4842072583588438 -0.01977906943199157 +42 -0.03756245021461515 0.4985853893301043 0.001293012151093093 +43 -0.03197562240286588 0.3292538121360879 0.3749259750481462 +44 -0.05717073009850145 0.1882774724090304 0.4596554155052142 +45 0.4463446586939401 0.2224449507821253 0.03598179438171378 +46 0.1140717386120424 0.3075061491743895 -0.3773958222741264 +47 0.236387555749274 0.430565411758525 0.09345774277231239 +48 0.3781159303516576 0.2921511032547155 0.1472279731618724 +49 0.06983315139943251 -0.1315243462511946 -0.4773098336602964 +50 -0.02765071572380252 0.3680778377326214 -0.3372745814466894 +51 -0.1320193234548694 0.4678916162372102 -0.1168260830869966 +52 -0.1785273372052953 0.2682171503236889 0.3823448053035721 +53 -0.1959825739858849 0.1177151785289119 0.4446728768856648 +54 -0.1887698711370119 0.4620978248460994 0.02883636633522927 +55 -0.3111183323264466 0.1859535959327892 0.344422187801651 +56 0.2985132750177531 -0.3645766738921073 -0.1672533213181233 +57 -0.2744647792092298 0.4088282694425724 -0.08676710827393114 +58 -0.3210765817368316 0.03378770063969206 0.381796044958135 +59 -0.4060689942095819 0.08905124879568629 0.2778090117860603 +60 -0.3963372268153362 -0.06581667877397752 0.2976322688074864 +61 0.1373446027596949 -0.2673875159255554 0.3995502176447192 +62 -0.4693534124375669 -0.007452357304369514 0.1721970865137363 +63 -0.438850244007533 -0.1604635574497901 0.1779379388019505 +64 -0.4882604819877454 -0.09782801685398738 0.04507084254271851 +65 -0.433483128474169 -0.244453321072111 0.0483213321946163 +66 -0.4606201007589651 -0.1732879745133468 -0.08831987695792189 +67 -0.3835118066267862 -0.3093567790002423 -0.08495338406710967 +68 -0.3367174993349611 -0.3659933267013772 0.05167407910809284 +69 -0.2678622816917925 -0.4145753666524341 -0.07985651765426952 +70 -0.205085219904653 -0.4522510217191782 0.05838720690887255 +71 -0.3894731974184563 -0.226083984285031 -0.2172479241384138 +72 -0.4452563648097808 -0.07999668223416313 -0.2129490559450916 +73 -0.003511259471652275 -0.2670358864798737 0.4227049874189217 +74 -0.3531697437392172 -0.1251144653684121 -0.3310853404528085 +75 -0.3889238884410299 0.02611817923218877 -0.3131390261744846 +76 -0.2787569096081184 -0.262391732725563 -0.3216289227401789 +77 -0.2494298444491921 -0.3975021742802097 0.1725594800077865 +78 -0.1252033142062692 -0.4784520517249065 -0.07353750275875261 +79 -0.05285823645319605 -0.493487744300911 0.06062881380875493 +80 0.2717315829336582 -0.2496002987006435 0.3374339012679639 +81 -0.4958471179395929 0.05427198185470963 0.03449909588034901 +82 0.3416362711514083 -0.3642358366721966 -0.02483774380788878 +83 -0.2782213803936585 -0.01651621960366377 -0.4151145359799518 +84 -0.3858944658937788 0.2313669119790785 0.2180706611028189 +85 0.1294227732117325 -0.2636823095586802 -0.4046249935433855 +86 0.1707978779883504 -0.3604532909633674 0.3014987726481727 +87 -0.2972397651044 0.1344755537844391 -0.3788995216083672 +88 -0.3907294249782021 0.1730813574975597 -0.2595637881966695 +89 0.03024010871706084 -0.4946174833247267 -0.06662642879739289 +90 -0.04471041513136528 -0.4559114631858334 -0.2003639601189132 +91 -0.2897813050045646 -0.1074806422588692 0.3930327045036742 +92 -0.2807274887421594 0.2760431923377526 -0.3082081001993674 +93 0.1060858997846735 -0.4830559581774032 0.07350321854304803 +94 -0.143288452953271 0.3805047259796351 0.2910061386937071 +95 -0.02077200627872767 -0.2621493593931338 -0.4252601993191064 +96 -0.07932822661777417 -0.1124931749608491 0.4806790176915431 +97 -0.2102294888032433 0.1047479542032376 -0.4413971319886021 +98 0.4350243525719355 -0.1278060359691311 0.2107591749823136 +99 0.2988398642836543 -0.3428866280534196 0.2076619748944975 +100 -0.3439615661858341 -0.2131697486180884 0.2936819695880035 +101 -0.2176566309176568 -0.2596656349182824 0.3676946410543132 +102 -0.318981501523703 0.3817633694742015 0.05007525749697323 +103 -0.3954450634832108 0.3035032899739587 -0.03884526665814639 +104 0.006956851727847007 -0.466794760772503 0.179037017204204 +105 -0.1417864340749447 -0.06392504578606578 -0.4751949027857525 +106 0.1233762020066791 -0.4458050737749751 -0.1898318966214479 +107 0.02732645725982314 -0.3620738512542798 -0.3437379684752919 +108 -0.1791788688258533 0.2313443603206242 -0.4054315230888305 +109 -0.1314425191005174 -0.3722853633163602 -0.3068003788017802 +110 0.2133449768024974 -0.4494436842688703 -0.04984270803202714 +111 -0.09541143508872867 0.4685689676979816 0.1460814175888131 +112 0.1955691103344901 -0.3493896792638685 -0.2994654823295451 +113 -0.2125797344507606 0.3929143036434359 -0.2245622552728707 +114 0.00442729082563781 0.4317905328294528 0.2520661319074867 +115 0.2514359214289099 -0.4240206590661072 0.0835850351458868 +116 -0.07168812674179963 0.03624030816997568 0.4935052710437821 +117 0.4097029578088054 -0.2505327347041654 0.1392007011633508 +118 0.3540488510939609 -0.1235910217552639 0.3307184155449105 +119 -0.4093296549558124 0.2697633070552225 0.09837170192888459 +120 -0.2377505688133197 0.4094324923466916 0.1607473210965559 +121 -0.2801677796377646 0.3103032612102715 0.2742588217999833 +122 0.03901999990292263 -0.3401769694293537 0.3643584348926706 +123 -0.1069554957395328 -0.3897746830201826 0.2943403105380352 +124 -0.1251520253506241 -0.4570701298706271 0.1594486341449332 +125 0.2502053496094969 0.4305271053809251 -0.04520724011827333 +126 -0.1586273363003201 -0.04494061596874992 0.4720357075631191 +127 -0.3721182374605039 -0.2835260141164211 0.1764681746620137 +128 -0.2352457422466886 -0.1521878196763474 -0.4141235423119097 +129 0.479986141812706 0.1246808144486656 -0.06378086057878406 +130 -0.07610425730749397 0.4319635319535118 -0.2400326000399157 +131 -0.2884530213086329 -0.353652776154199 -0.2042659257349335 +132 -0.4588401370156111 0.05746394149401464 -0.190167358112447 +133 -0.4440831055175186 0.1949204155884832 -0.1216397426038441 +134 -0.1780552490608454 -0.43316536366806 -0.1751116672303638 +135 -0.4919631731655178 -0.02985239287932549 -0.08414909915318818 +136 -0.4563613828471496 0.1383966100270178 0.1502686480236875 +137 -0.3459047159723113 0.3134150147362468 -0.1792232016397231 +138 -0.1531697265302356 -0.269052072762942 -0.3926194302586379 +139 -0.04199239465102653 0.170189522618944 -0.4682650586817346 +140 0.2436120704180919 0.2419819558249213 -0.3634527372324448 +141 0.3388675763450887 0.2526784722646921 0.2670624559087006 +142 0.426863022631354 -0.2599402590848207 0.01479937893765846 +143 0.1583421021825986 -0.434609327079221 0.1898486541752199 +144 0.04529312439054804 0.09527644750198475 0.4887442392850722 +145 -0.237259629761916 0.01661594150769146 0.4398087977440328 +146 0.2471756661872414 -0.2384733444196505 -0.3633657304234952 +147 0.0577997827759179 0.4843451626752797 0.1098587661687201 +148 -0.3347709242399347 0.3325740862996533 0.1652964167956783 +149 -0.1531835821016216 0.3394123597208809 -0.3336675594708289 +150 0.3678596011663183 -0.2287923206847306 0.2496665532774717 +151 0.4833905865252366 0.1023321567369471 0.07656154750494946 +152 0.139648641361451 0.213951152339616 -0.4297943245070266 +153 -0.03588640342916357 0.3020322016399679 -0.3968484789203974 +154 -0.1807867118628844 -0.175443810495746 0.431897712627121 +155 0.3482150266595301 -0.3467237776036438 0.09235213723981545 +156 -0.1144804135533086 -0.2906546149491346 0.3904025226883578 +157 -0.272442803422386 -0.3214727720944405 0.2691285485883411 +158 -0.4661144982963306 0.1808181207555283 0.006487039725418148 +159 -0.1294151845848306 -0.2035855719987957 -0.4379550489180338 +160 0.0643376427947949 -0.4125758147354672 0.2750306615906213 +161 -0.485934856252322 0.09063649334671869 -0.07518205605625157 +162 -1.054711873393899e-15 2.740863092043355e-16 1.387778780781446e-17 +163 -0.2518287427575621 0.01394909067848053 -0.01813384063889455 +164 0.1979966627205619 -0.07912117779450417 0.136432807235424 +165 0.1252557772189523 0.2108310184636258 -0.06396909785024807 +166 -0.1396282365634093 -0.2126533564490596 0.005451212559497404 +167 -0.1452236908860622 0.1899245943515788 0.1115930546284099 +168 -0.1128975635045348 -0.03739761584310859 -0.2239666615944753 +169 0.2255282581475765 -0.02324317013777008 -0.114912387106339 +170 -0.1185300984451474 -0.06898069323449257 0.2117841676382997 +171 0.09826973804035274 0.1479593924849172 0.1780218172594331 +172 0.1008522355738519 -0.2194506497096799 -0.0387973024559414 +173 -0.1004802078206753 0.1922847571405704 -0.1489913450586494 +174 0.08807336311451003 0.05169962805137276 -0.2710619397211838 +175 0.2721130716947845 0.1012440132503971 0.05125758379007644 +176 0.03417206272185144 -0.2215719151561445 0.170600381517002 +177 0.1137937440633116 -0.1504669188799527 -0.2328308003328481 +178 0.06859239970200379 -0.02458786916204119 0.2886797905418265 +179 -0.07412101863376669 0.1115946146904476 0.2784077081876079 +180 -0.03510942034185002 0.3035187048788698 -0.002615213197338789 +181 -0.2584473439251872 0.04951118726336353 0.1670864992119518 +182 -0.271439309774019 -0.1253498071949041 0.1040529980285466 +183 -0.04852043731728409 -0.2508916955712664 -0.1696491742543922 +184 -0.2454089059542846 -0.1370648073924176 -0.1323370643966134 +185 -0.2475613240140267 0.1956305775781199 -0.04007129924591253 +186 -0.2454894510325761 0.08553979857837071 -0.1877092783560849 +187 0.2725465624551494 -0.1639927600175499 -0.007661258060314396 +188 0.2737169869289461 0.1495048171705471 -0.1364802343336155 +189 0.05253349228003979 0.2452147643184821 -0.2225910396278519 +190 -0.02545322226002612 0.2650727312958021 0.2127802314180369 +191 0.1018834233771898 0.3032469532876146 0.1025401607002092 +192 -0.1415747878459704 -0.2417761394138328 0.1801947156135768 +193 -0.01216178176390184 -0.3258296686301808 0.03946342601621335 +194 0.2226199802004204 0.04389900870267784 0.2541816686302036 +195 0.1878620048199119 -0.2655600924779695 0.1018767956671858 +196 0.3422393613961585 -0.04241489652435964 0.07387300453144992 +197 -0.1204663389458891 0.1145805059910938 -0.297963959751455 +198 0.1515440630746299 -0.1636412545963076 0.2681085820708983 +$EndNodes +$Elements +966 +1 15 2 0 1 1 +2 15 2 0 2 2 +3 1 2 0 2 2 3 +4 1 2 0 2 3 4 +5 1 2 0 2 4 5 +6 1 2 0 2 5 6 +7 1 2 0 2 6 7 +8 1 2 0 2 7 8 +9 1 2 0 2 8 9 +10 1 2 0 2 9 10 +11 1 2 0 2 10 11 +12 1 2 0 2 11 1 +13 2 2 0 1 1 25 11 +14 2 2 0 1 11 144 1 +15 2 2 0 1 1 96 25 +16 2 2 0 1 1 116 96 +17 2 2 0 1 1 144 116 +18 2 2 0 1 2 26 3 +19 2 2 0 1 3 49 2 +20 2 2 0 1 2 139 26 +21 2 2 0 1 49 105 2 +22 2 2 0 1 105 139 2 +23 2 2 0 1 4 14 3 +24 2 2 0 1 3 19 4 +25 2 2 0 1 14 49 3 +26 2 2 0 1 3 26 19 +27 2 2 0 1 5 24 4 +28 2 2 0 1 4 27 5 +29 2 2 0 1 4 24 14 +30 2 2 0 1 19 27 4 +31 2 2 0 1 5 17 6 +32 2 2 0 1 6 18 5 +33 2 2 0 1 5 27 17 +34 2 2 0 1 18 24 5 +35 2 2 0 1 7 21 6 +36 2 2 0 1 6 129 7 +37 2 2 0 1 17 129 6 +38 2 2 0 1 6 21 18 +39 2 2 0 1 8 13 7 +40 2 2 0 1 7 151 8 +41 2 2 0 1 13 21 7 +42 2 2 0 1 129 151 7 +43 2 2 0 1 8 16 9 +44 2 2 0 1 9 98 8 +45 2 2 0 1 8 98 13 +46 2 2 0 1 8 151 16 +47 2 2 0 1 9 20 10 +48 2 2 0 1 10 118 9 +49 2 2 0 1 16 20 9 +50 2 2 0 1 9 118 98 +51 2 2 0 1 10 12 11 +52 2 2 0 1 11 15 10 +53 2 2 0 1 10 20 12 +54 2 2 0 1 15 118 10 +55 2 2 0 1 12 144 11 +56 2 2 0 1 11 25 15 +57 2 2 0 1 20 29 12 +58 2 2 0 1 29 30 12 +59 2 2 0 1 30 144 12 +60 2 2 0 1 13 142 21 +61 2 2 0 1 98 117 13 +62 2 2 0 1 117 142 13 +63 2 2 0 1 24 146 14 +64 2 2 0 1 14 85 49 +65 2 2 0 1 14 146 85 +66 2 2 0 1 25 61 15 +67 2 2 0 1 61 80 15 +68 2 2 0 1 80 118 15 +69 2 2 0 1 16 141 20 +70 2 2 0 1 45 48 16 +71 2 2 0 1 16 151 45 +72 2 2 0 1 48 141 16 +73 2 2 0 1 27 28 17 +74 2 2 0 1 28 31 17 +75 2 2 0 1 31 129 17 +76 2 2 0 1 21 22 18 +77 2 2 0 1 22 23 18 +78 2 2 0 1 23 24 18 +79 2 2 0 1 26 152 19 +80 2 2 0 1 19 140 27 +81 2 2 0 1 19 152 140 +82 2 2 0 1 20 141 29 +83 2 2 0 1 21 142 22 +84 2 2 0 1 22 56 23 +85 2 2 0 1 22 82 56 +86 2 2 0 1 22 142 82 +87 2 2 0 1 23 146 24 +88 2 2 0 1 56 112 23 +89 2 2 0 1 112 146 23 +90 2 2 0 1 25 73 61 +91 2 2 0 1 25 96 73 +92 2 2 0 1 139 152 26 +93 2 2 0 1 27 140 28 +94 2 2 0 1 28 32 31 +95 2 2 0 1 28 34 32 +96 2 2 0 1 28 140 34 +97 2 2 0 1 29 33 30 +98 2 2 0 1 29 38 33 +99 2 2 0 1 29 141 38 +100 2 2 0 1 33 43 30 +101 2 2 0 1 43 44 30 +102 2 2 0 1 44 144 30 +103 2 2 0 1 32 40 31 +104 2 2 0 1 40 45 31 +105 2 2 0 1 45 129 31 +106 2 2 0 1 34 35 32 +107 2 2 0 1 35 125 32 +108 2 2 0 1 32 125 40 +109 2 2 0 1 38 39 33 +110 2 2 0 1 39 114 33 +111 2 2 0 1 33 114 43 +112 2 2 0 1 34 36 35 +113 2 2 0 1 34 46 36 +114 2 2 0 1 34 140 46 +115 2 2 0 1 36 37 35 +116 2 2 0 1 37 41 35 +117 2 2 0 1 41 125 35 +118 2 2 0 1 36 130 37 +119 2 2 0 1 46 50 36 +120 2 2 0 1 50 130 36 +121 2 2 0 1 37 42 41 +122 2 2 0 1 37 51 42 +123 2 2 0 1 37 130 51 +124 2 2 0 1 38 47 39 +125 2 2 0 1 38 48 47 +126 2 2 0 1 38 141 48 +127 2 2 0 1 47 147 39 +128 2 2 0 1 39 147 114 +129 2 2 0 1 40 48 45 +130 2 2 0 1 47 48 40 +131 2 2 0 1 40 125 47 +132 2 2 0 1 42 147 41 +133 2 2 0 1 47 125 41 +134 2 2 0 1 41 147 47 +135 2 2 0 1 51 54 42 +136 2 2 0 1 54 111 42 +137 2 2 0 1 111 147 42 +138 2 2 0 1 43 52 44 +139 2 2 0 1 43 94 52 +140 2 2 0 1 43 114 94 +141 2 2 0 1 52 53 44 +142 2 2 0 1 53 116 44 +143 2 2 0 1 116 144 44 +144 2 2 0 1 45 151 129 +145 2 2 0 1 46 153 50 +146 2 2 0 1 140 152 46 +147 2 2 0 1 152 153 46 +148 2 2 0 1 85 95 49 +149 2 2 0 1 95 159 49 +150 2 2 0 1 49 159 105 +151 2 2 0 1 50 149 130 +152 2 2 0 1 50 153 149 +153 2 2 0 1 51 57 54 +154 2 2 0 1 51 113 57 +155 2 2 0 1 51 130 113 +156 2 2 0 1 52 55 53 +157 2 2 0 1 52 121 55 +158 2 2 0 1 94 121 52 +159 2 2 0 1 55 58 53 +160 2 2 0 1 58 145 53 +161 2 2 0 1 53 145 116 +162 2 2 0 1 57 102 54 +163 2 2 0 1 102 120 54 +164 2 2 0 1 54 120 111 +165 2 2 0 1 55 59 58 +166 2 2 0 1 55 84 59 +167 2 2 0 1 55 121 84 +168 2 2 0 1 82 110 56 +169 2 2 0 1 56 110 106 +170 2 2 0 1 106 112 56 +171 2 2 0 1 57 103 102 +172 2 2 0 1 57 137 103 +173 2 2 0 1 113 137 57 +174 2 2 0 1 59 60 58 +175 2 2 0 1 60 91 58 +176 2 2 0 1 91 145 58 +177 2 2 0 1 59 62 60 +178 2 2 0 1 59 136 62 +179 2 2 0 1 84 136 59 +180 2 2 0 1 62 63 60 +181 2 2 0 1 63 100 60 +182 2 2 0 1 60 100 91 +183 2 2 0 1 73 122 61 +184 2 2 0 1 61 86 80 +185 2 2 0 1 61 122 86 +186 2 2 0 1 62 64 63 +187 2 2 0 1 62 81 64 +188 2 2 0 1 62 136 81 +189 2 2 0 1 64 65 63 +190 2 2 0 1 65 127 63 +191 2 2 0 1 63 127 100 +192 2 2 0 1 64 66 65 +193 2 2 0 1 64 135 66 +194 2 2 0 1 81 135 64 +195 2 2 0 1 66 67 65 +196 2 2 0 1 67 68 65 +197 2 2 0 1 68 127 65 +198 2 2 0 1 66 71 67 +199 2 2 0 1 66 72 71 +200 2 2 0 1 66 135 72 +201 2 2 0 1 67 69 68 +202 2 2 0 1 67 131 69 +203 2 2 0 1 71 131 67 +204 2 2 0 1 69 70 68 +205 2 2 0 1 70 77 68 +206 2 2 0 1 77 127 68 +207 2 2 0 1 69 78 70 +208 2 2 0 1 69 134 78 +209 2 2 0 1 131 134 69 +210 2 2 0 1 70 124 77 +211 2 2 0 1 78 79 70 +212 2 2 0 1 79 124 70 +213 2 2 0 1 72 74 71 +214 2 2 0 1 74 76 71 +215 2 2 0 1 76 131 71 +216 2 2 0 1 72 75 74 +217 2 2 0 1 72 132 75 +218 2 2 0 1 72 135 132 +219 2 2 0 1 96 156 73 +220 2 2 0 1 73 156 122 +221 2 2 0 1 75 83 74 +222 2 2 0 1 74 128 76 +223 2 2 0 1 83 128 74 +224 2 2 0 1 75 87 83 +225 2 2 0 1 75 88 87 +226 2 2 0 1 75 132 88 +227 2 2 0 1 109 131 76 +228 2 2 0 1 76 138 109 +229 2 2 0 1 128 138 76 +230 2 2 0 1 77 124 123 +231 2 2 0 1 123 157 77 +232 2 2 0 1 77 157 127 +233 2 2 0 1 78 89 79 +234 2 2 0 1 78 90 89 +235 2 2 0 1 78 134 90 +236 2 2 0 1 89 93 79 +237 2 2 0 1 93 104 79 +238 2 2 0 1 104 124 79 +239 2 2 0 1 86 99 80 +240 2 2 0 1 99 150 80 +241 2 2 0 1 80 150 118 +242 2 2 0 1 81 161 135 +243 2 2 0 1 136 158 81 +244 2 2 0 1 158 161 81 +245 2 2 0 1 82 115 110 +246 2 2 0 1 82 155 115 +247 2 2 0 1 142 155 82 +248 2 2 0 1 87 97 83 +249 2 2 0 1 97 105 83 +250 2 2 0 1 105 128 83 +251 2 2 0 1 119 136 84 +252 2 2 0 1 84 148 119 +253 2 2 0 1 121 148 84 +254 2 2 0 1 85 107 95 +255 2 2 0 1 85 112 107 +256 2 2 0 1 85 146 112 +257 2 2 0 1 86 143 99 +258 2 2 0 1 122 160 86 +259 2 2 0 1 86 160 143 +260 2 2 0 1 88 92 87 +261 2 2 0 1 92 108 87 +262 2 2 0 1 87 108 97 +263 2 2 0 1 88 137 92 +264 2 2 0 1 132 133 88 +265 2 2 0 1 133 137 88 +266 2 2 0 1 90 106 89 +267 2 2 0 1 89 110 93 +268 2 2 0 1 106 110 89 +269 2 2 0 1 90 107 106 +270 2 2 0 1 90 109 107 +271 2 2 0 1 90 134 109 +272 2 2 0 1 100 101 91 +273 2 2 0 1 101 154 91 +274 2 2 0 1 126 145 91 +275 2 2 0 1 91 154 126 +276 2 2 0 1 92 149 108 +277 2 2 0 1 92 137 113 +278 2 2 0 1 113 149 92 +279 2 2 0 1 93 143 104 +280 2 2 0 1 110 115 93 +281 2 2 0 1 115 143 93 +282 2 2 0 1 94 114 111 +283 2 2 0 1 111 120 94 +284 2 2 0 1 120 121 94 +285 2 2 0 1 107 138 95 +286 2 2 0 1 138 159 95 +287 2 2 0 1 116 126 96 +288 2 2 0 1 126 154 96 +289 2 2 0 1 154 156 96 +290 2 2 0 1 97 139 105 +291 2 2 0 1 108 139 97 +292 2 2 0 1 98 150 117 +293 2 2 0 1 118 150 98 +294 2 2 0 1 99 143 115 +295 2 2 0 1 115 155 99 +296 2 2 0 1 117 150 99 +297 2 2 0 1 99 155 117 +298 2 2 0 1 100 157 101 +299 2 2 0 1 127 157 100 +300 2 2 0 1 123 156 101 +301 2 2 0 1 101 157 123 +302 2 2 0 1 101 156 154 +303 2 2 0 1 103 119 102 +304 2 2 0 1 119 148 102 +305 2 2 0 1 102 148 120 +306 2 2 0 1 103 158 119 +307 2 2 0 1 103 137 133 +308 2 2 0 1 133 158 103 +309 2 2 0 1 123 124 104 +310 2 2 0 1 104 160 123 +311 2 2 0 1 143 160 104 +312 2 2 0 1 105 159 128 +313 2 2 0 1 107 112 106 +314 2 2 0 1 109 138 107 +315 2 2 0 1 108 153 139 +316 2 2 0 1 149 153 108 +317 2 2 0 1 109 134 131 +318 2 2 0 1 114 147 111 +319 2 2 0 1 130 149 113 +320 2 2 0 1 116 145 126 +321 2 2 0 1 117 155 142 +322 2 2 0 1 119 158 136 +323 2 2 0 1 120 148 121 +324 2 2 0 1 122 156 123 +325 2 2 0 1 123 160 122 +326 2 2 0 1 128 159 138 +327 2 2 0 1 132 161 133 +328 2 2 0 1 135 161 132 +329 2 2 0 1 133 161 158 +330 2 2 0 1 139 153 152 +331 4 2 0 1 166 170 163 182 +332 4 2 0 1 165 162 171 180 +333 4 2 0 1 167 171 162 180 +334 4 2 0 1 4 5 27 169 +335 4 2 0 1 163 166 162 170 +336 4 2 0 1 4 24 169 177 +337 4 2 0 1 167 173 163 185 +338 4 2 0 1 162 163 167 173 +339 4 2 0 1 158 136 181 185 +340 4 2 0 1 162 171 170 178 +341 4 2 0 1 162 163 170 181 +342 4 2 0 1 162 167 163 181 +343 4 2 0 1 4 169 27 174 +344 4 2 0 1 170 171 162 179 +345 4 2 0 1 162 173 165 174 +346 4 2 0 1 170 178 171 179 +347 4 2 0 1 158 181 163 185 +348 4 2 0 1 162 176 164 178 +349 4 2 0 1 162 170 167 181 +350 4 2 0 1 162 170 176 178 +351 4 2 0 1 162 166 163 184 +352 4 2 0 1 167 170 162 179 +353 4 2 0 1 129 169 6 196 +354 4 2 0 1 162 165 169 174 +355 4 2 0 1 73 176 170 178 +356 4 2 0 1 119 181 136 185 +357 4 2 0 1 4 169 174 177 +358 4 2 0 1 72 132 135 163 +359 4 2 0 1 27 174 169 188 +360 4 2 0 1 158 119 136 185 +361 4 2 0 1 64 66 163 182 +362 4 2 0 1 167 162 171 179 +363 4 2 0 1 66 135 64 163 +364 4 2 0 1 7 129 6 196 +365 4 2 0 1 162 173 167 180 +366 4 2 0 1 95 177 168 183 +367 4 2 0 1 162 177 172 183 +368 4 2 0 1 4 24 5 169 +369 4 2 0 1 27 169 5 188 +370 4 2 0 1 162 169 165 175 +371 4 2 0 1 162 164 169 175 +372 4 2 0 1 129 175 169 196 +373 4 2 0 1 168 177 162 183 +374 4 2 0 1 132 161 135 163 +375 4 2 0 1 119 148 84 167 +376 4 2 0 1 162 165 173 180 +377 4 2 0 1 164 162 171 175 +378 4 2 0 1 165 171 162 175 +379 4 2 0 1 158 81 161 163 +380 4 2 0 1 30 171 178 179 +381 4 2 0 1 64 81 62 163 +382 4 2 0 1 169 162 174 177 +383 4 2 0 1 66 163 182 184 +384 4 2 0 1 169 177 23 187 +385 4 2 0 1 162 170 166 176 +386 4 2 0 1 54 167 102 185 +387 4 2 0 1 68 166 67 182 +388 4 2 0 1 162 168 163 173 +389 4 2 0 1 164 171 162 178 +390 4 2 0 1 161 158 163 185 +391 4 2 0 1 65 68 67 182 +392 4 2 0 1 166 168 162 183 +393 4 2 0 1 129 6 169 188 +394 4 2 0 1 162 163 168 184 +395 4 2 0 1 165 175 40 191 +396 4 2 0 1 129 169 175 188 +397 4 2 0 1 119 167 84 181 +398 4 2 0 1 62 64 163 182 +399 4 2 0 1 18 169 23 187 +400 4 2 0 1 67 182 166 184 +401 4 2 0 1 162 168 166 184 +402 4 2 0 1 168 174 162 177 +403 4 2 0 1 164 162 169 172 +404 4 2 0 1 167 181 119 185 +405 4 2 0 1 126 178 170 179 +406 4 2 0 1 72 163 135 184 +407 4 2 0 1 72 132 163 186 +408 4 2 0 1 54 180 167 185 +409 4 2 0 1 62 163 81 181 +410 4 2 0 1 162 168 173 174 +411 4 2 0 1 66 135 163 184 +412 4 2 0 1 165 174 173 189 +413 4 2 0 1 38 141 171 191 +414 4 2 0 1 140 174 188 189 +415 4 2 0 1 161 163 132 186 +416 4 2 0 1 24 23 18 169 +417 4 2 0 1 62 163 181 182 +418 4 2 0 1 65 182 67 184 +419 4 2 0 1 166 162 172 183 +420 4 2 0 1 140 174 27 188 +421 4 2 0 1 162 166 172 176 +422 4 2 0 1 4 14 24 177 +423 4 2 0 1 23 169 24 177 +424 4 2 0 1 169 172 162 177 +425 4 2 0 1 163 173 168 186 +426 4 2 0 1 60 181 170 182 +427 4 2 0 1 119 148 167 185 +428 4 2 0 1 57 173 51 180 +429 4 2 0 1 72 163 184 186 +430 4 2 0 1 41 35 165 180 +431 4 2 0 1 41 37 35 180 +432 4 2 0 1 141 171 48 175 +433 4 2 0 1 56 177 172 187 +434 4 2 0 1 135 161 81 163 +435 4 2 0 1 95 49 168 177 +436 4 2 0 1 102 54 120 167 +437 4 2 0 1 34 140 188 189 +438 4 2 0 1 176 178 73 198 +439 4 2 0 1 162 172 164 176 +440 4 2 0 1 141 171 175 194 +441 4 2 0 1 60 170 91 182 +442 4 2 0 1 130 37 180 189 +443 4 2 0 1 56 23 177 187 +444 4 2 0 1 22 18 23 187 +445 4 2 0 1 121 55 167 179 +446 4 2 0 1 6 169 21 196 +447 4 2 0 1 60 91 100 182 +448 4 2 0 1 12 30 171 178 +449 4 2 0 1 65 66 182 184 +450 4 2 0 1 91 170 100 182 +451 4 2 0 1 164 172 169 187 +452 4 2 0 1 116 178 126 179 +453 4 2 0 1 125 165 40 191 +454 4 2 0 1 69 67 166 184 +455 4 2 0 1 96 73 170 178 +456 4 2 0 1 113 51 57 173 +457 4 2 0 1 57 173 180 185 +458 4 2 0 1 136 119 84 181 +459 4 2 0 1 84 167 121 181 +460 4 2 0 1 83 75 74 186 +461 4 2 0 1 134 131 69 183 +462 4 2 0 1 167 179 55 181 +463 4 2 0 1 130 180 173 189 +464 4 2 0 1 145 179 170 181 +465 4 2 0 1 48 171 141 191 +466 4 2 0 1 48 175 171 191 +467 4 2 0 1 102 167 148 185 +468 4 2 0 1 64 135 81 163 +469 4 2 0 1 30 178 144 179 +470 4 2 0 1 57 180 54 185 +471 4 2 0 1 121 52 55 179 +472 4 2 0 1 40 175 165 188 +473 4 2 0 1 60 91 170 181 +474 4 2 0 1 130 51 173 180 +475 4 2 0 1 76 183 168 184 +476 4 2 0 1 74 75 72 184 +477 4 2 0 1 140 19 27 174 +478 4 2 0 1 165 180 171 191 +479 4 2 0 1 49 174 168 177 +480 4 2 0 1 83 74 168 186 +481 4 2 0 1 84 121 55 181 +482 4 2 0 1 69 70 78 166 +483 4 2 0 1 120 167 54 180 +484 4 2 0 1 121 167 55 181 +485 4 2 0 1 148 121 84 167 +486 4 2 0 1 74 128 168 184 +487 4 2 0 1 131 166 69 183 +488 4 2 0 1 156 170 176 192 +489 4 2 0 1 74 75 184 186 +490 4 2 0 1 67 68 69 166 +491 4 2 0 1 65 67 66 184 +492 4 2 0 1 117 164 150 196 +493 4 2 0 1 96 156 154 170 +494 4 2 0 1 98 150 164 196 +495 4 2 0 1 145 170 58 181 +496 4 2 0 1 161 133 158 185 +497 4 2 0 1 24 18 5 169 +498 4 2 0 1 78 183 166 193 +499 4 2 0 1 34 32 165 188 +500 4 2 0 1 98 117 150 196 +501 4 2 0 1 41 35 125 165 +502 4 2 0 1 35 34 32 165 +503 4 2 0 1 73 156 170 176 +504 4 2 0 1 74 184 168 186 +505 4 2 0 1 72 184 75 186 +506 4 2 0 1 133 161 132 186 +507 4 2 0 1 119 102 148 185 +508 4 2 0 1 150 164 117 195 +509 4 2 0 1 34 188 165 189 +510 4 2 0 1 57 54 102 185 +511 4 2 0 1 148 102 120 167 +512 4 2 0 1 170 181 163 182 +513 4 2 0 1 12 178 171 194 +514 4 2 0 1 6 18 21 169 +515 4 2 0 1 5 169 6 188 +516 4 2 0 1 136 62 81 181 +517 4 2 0 1 171 180 167 190 +518 4 2 0 1 58 170 91 181 +519 4 2 0 1 78 166 70 193 +520 4 2 0 1 2 168 49 174 +521 4 2 0 1 34 140 28 188 +522 4 2 0 1 40 125 32 165 +523 4 2 0 1 38 48 141 191 +524 4 2 0 1 169 174 165 188 +525 4 2 0 1 40 175 48 191 +526 4 2 0 1 113 130 51 173 +527 4 2 0 1 66 72 135 184 +528 4 2 0 1 169 187 21 196 +529 4 2 0 1 131 166 183 184 +530 4 2 0 1 74 128 83 168 +531 4 2 0 1 18 21 169 187 +532 4 2 0 1 7 6 21 196 +533 4 2 0 1 64 65 66 182 +534 4 2 0 1 69 166 131 184 +535 4 2 0 1 56 112 172 177 +536 4 2 0 1 76 168 128 184 +537 4 2 0 1 166 182 68 192 +538 4 2 0 1 159 95 168 183 +539 4 2 0 1 34 165 35 189 +540 4 2 0 1 96 170 126 178 +541 4 2 0 1 172 183 89 193 +542 4 2 0 1 150 118 98 164 +543 4 2 0 1 172 177 106 183 +544 4 2 0 1 168 183 166 184 +545 4 2 0 1 111 167 120 180 +546 4 2 0 1 17 5 6 188 +547 4 2 0 1 30 171 179 190 +548 4 2 0 1 130 37 51 180 +549 4 2 0 1 40 165 32 188 +550 4 2 0 1 100 182 170 192 +551 4 2 0 1 112 106 172 177 +552 4 2 0 1 74 76 128 184 +553 4 2 0 1 170 179 167 181 +554 4 2 0 1 32 125 35 165 +555 4 2 0 1 41 165 125 191 +556 4 2 0 1 113 173 57 185 +557 4 2 0 1 2 168 174 197 +558 4 2 0 1 18 6 5 169 +559 4 2 0 1 167 180 111 190 +560 4 2 0 1 126 170 145 179 +561 4 2 0 1 12 144 30 178 +562 4 2 0 1 91 126 154 170 +563 4 2 0 1 69 68 70 166 +564 4 2 0 1 4 27 19 174 +565 4 2 0 1 137 92 173 186 +566 4 2 0 1 25 73 178 198 +567 4 2 0 1 29 171 141 194 +568 4 2 0 1 60 58 91 181 +569 4 2 0 1 120 167 111 190 +570 4 2 0 1 111 120 54 180 +571 4 2 0 1 150 118 164 198 +572 4 2 0 1 30 33 29 171 +573 4 2 0 1 69 131 67 184 +574 4 2 0 1 95 49 159 168 +575 4 2 0 1 68 182 127 192 +576 4 2 0 1 117 187 164 196 +577 4 2 0 1 166 176 170 192 +578 4 2 0 1 33 171 30 190 +579 4 2 0 1 57 51 54 180 +580 4 2 0 1 137 92 113 173 +581 4 2 0 1 22 23 56 187 +582 4 2 0 1 37 36 35 189 +583 4 2 0 1 29 38 141 171 +584 4 2 0 1 150 164 195 198 +585 4 2 0 1 180 190 171 191 +586 4 2 0 1 145 91 58 170 +587 4 2 0 1 41 180 165 191 +588 4 2 0 1 82 187 172 195 +589 4 2 0 1 137 173 185 186 +590 4 2 0 1 113 137 173 185 +591 4 2 0 1 112 106 177 183 +592 4 2 0 1 164 187 117 195 +593 4 2 0 1 154 170 156 192 +594 4 2 0 1 148 120 121 167 +595 4 2 0 1 165 188 174 189 +596 4 2 0 1 76 138 168 183 +597 4 2 0 1 116 96 126 178 +598 4 2 0 1 76 138 128 168 +599 4 2 0 1 82 56 172 187 +600 4 2 0 1 172 176 166 193 +601 4 2 0 1 9 98 118 164 +602 4 2 0 1 30 29 12 171 +603 4 2 0 1 70 166 192 193 +604 4 2 0 1 33 38 29 171 +605 4 2 0 1 165 175 169 188 +606 4 2 0 1 96 73 156 170 +607 4 2 0 1 141 175 16 194 +608 4 2 0 1 96 25 73 178 +609 4 2 0 1 45 175 40 188 +610 4 2 0 1 98 164 9 196 +611 4 2 0 1 121 94 52 190 +612 4 2 0 1 106 56 112 172 +613 4 2 0 1 70 68 77 166 +614 4 2 0 1 70 192 124 193 +615 4 2 0 1 169 175 164 196 +616 4 2 0 1 31 45 40 188 +617 4 2 0 1 172 177 169 187 +618 4 2 0 1 106 89 172 183 +619 4 2 0 1 113 149 130 173 +620 4 2 0 1 149 189 173 197 +621 4 2 0 1 121 167 94 190 +622 4 2 0 1 171 175 165 191 +623 4 2 0 1 93 110 193 195 +624 4 2 0 1 172 193 110 195 +625 4 2 0 1 125 40 47 191 +626 4 2 0 1 3 49 14 177 +627 4 2 0 1 96 154 126 170 +628 4 2 0 1 78 90 183 193 +629 4 2 0 1 152 26 174 189 +630 4 2 0 1 113 57 137 185 +631 4 2 0 1 16 194 175 196 +632 4 2 0 1 60 62 181 182 +633 4 2 0 1 110 82 172 195 +634 4 2 0 1 92 149 113 173 +635 4 2 0 1 146 23 24 177 +636 4 2 0 1 34 28 32 188 +637 4 2 0 1 163 185 173 186 +638 4 2 0 1 72 75 132 186 +639 4 2 0 1 91 145 126 170 +640 4 2 0 1 38 171 33 191 +641 4 2 0 1 124 192 104 193 +642 4 2 0 1 18 22 21 187 +643 4 2 0 1 77 166 68 192 +644 4 2 0 1 130 36 37 189 +645 4 2 0 1 30 179 43 190 +646 4 2 0 1 131 183 76 184 +647 4 2 0 1 89 110 172 193 +648 4 2 0 1 121 120 94 167 +649 4 2 0 1 9 164 118 194 +650 4 2 0 1 173 174 168 197 +651 4 2 0 1 12 171 29 194 +652 4 2 0 1 56 23 112 177 +653 4 2 0 1 116 1 96 178 +654 4 2 0 1 17 27 5 188 +655 4 2 0 1 140 27 28 188 +656 4 2 0 1 104 192 176 193 +657 4 2 0 1 120 94 167 190 +658 4 2 0 1 112 177 107 183 +659 4 2 0 1 82 56 110 172 +660 4 2 0 1 169 164 187 196 +661 4 2 0 1 164 175 171 194 +662 4 2 0 1 118 194 164 198 +663 4 2 0 1 3 174 49 177 +664 4 2 0 1 9 164 194 196 +665 4 2 0 1 70 166 77 192 +666 4 2 0 1 167 180 173 185 +667 4 2 0 1 146 24 14 177 +668 4 2 0 1 30 44 43 179 +669 4 2 0 1 101 154 156 192 +670 4 2 0 1 152 174 140 189 +671 4 2 0 1 166 182 163 184 +672 4 2 0 1 92 149 173 197 +673 4 2 0 1 171 190 33 191 +674 4 2 0 1 60 63 62 182 +675 4 2 0 1 45 129 175 188 +676 4 2 0 1 163 181 167 185 +677 4 2 0 1 150 195 99 198 +678 4 2 0 1 167 179 171 190 +679 4 2 0 1 143 193 176 195 +680 4 2 0 1 164 176 172 195 +681 4 2 0 1 171 178 164 194 +682 4 2 0 1 164 178 176 198 +683 4 2 0 1 9 194 16 196 +684 4 2 0 1 129 17 6 188 +685 4 2 0 1 78 89 90 193 +686 4 2 0 1 10 118 15 194 +687 4 2 0 1 10 15 178 194 +688 4 2 0 1 150 99 80 198 +689 4 2 0 1 83 186 168 197 +690 4 2 0 1 111 190 180 191 +691 4 2 0 1 77 68 127 192 +692 4 2 0 1 127 68 65 182 +693 4 2 0 1 118 15 194 198 +694 4 2 0 1 10 15 11 178 +695 4 2 0 1 19 3 4 174 +696 4 2 0 1 9 16 8 196 +697 4 2 0 1 45 31 129 188 +698 4 2 0 1 30 144 44 179 +699 4 2 0 1 149 153 189 197 +700 4 2 0 1 40 32 31 188 +701 4 2 0 1 159 168 138 183 +702 4 2 0 1 124 123 104 192 +703 4 2 0 1 89 183 90 193 +704 4 2 0 1 104 176 143 193 +705 4 2 0 1 62 63 64 182 +706 4 2 0 1 137 185 133 186 +707 4 2 0 1 178 194 15 198 +708 4 2 0 1 47 40 48 191 +709 4 2 0 1 89 110 106 172 +710 4 2 0 1 75 83 87 186 +711 4 2 0 1 123 176 104 192 +712 4 2 0 1 105 49 2 168 +713 4 2 0 1 107 177 95 183 +714 4 2 0 1 87 186 83 197 +715 4 2 0 1 173 186 92 197 +716 4 2 0 1 73 122 176 198 +717 4 2 0 1 159 138 95 183 +718 4 2 0 1 149 153 50 189 +719 4 2 0 1 112 107 106 183 +720 4 2 0 1 137 133 88 186 +721 4 2 0 1 120 111 94 190 +722 4 2 0 1 110 56 106 172 +723 4 2 0 1 93 193 143 195 +724 4 2 0 1 170 182 166 192 +725 4 2 0 1 173 180 165 189 +726 4 2 0 1 10 178 11 194 +727 4 2 0 1 114 190 111 191 +728 4 2 0 1 52 53 55 179 +729 4 2 0 1 173 189 174 197 +730 4 2 0 1 139 174 26 189 +731 4 2 0 1 19 140 152 174 +732 4 2 0 1 116 145 53 179 +733 4 2 0 1 36 34 35 189 +734 4 2 0 1 100 127 182 192 +735 4 2 0 1 116 126 145 179 +736 4 2 0 1 134 78 90 183 +737 4 2 0 1 112 23 146 177 +738 4 2 0 1 89 93 110 193 +739 4 2 0 1 115 110 93 195 +740 4 2 0 1 63 60 100 182 +741 4 2 0 1 159 128 138 168 +742 4 2 0 1 11 144 12 178 +743 4 2 0 1 46 140 34 189 +744 4 2 0 1 168 184 163 186 +745 4 2 0 1 16 175 151 196 +746 4 2 0 1 106 90 89 183 +747 4 2 0 1 76 131 109 183 +748 4 2 0 1 41 42 37 180 +749 4 2 0 1 2 3 26 174 +750 4 2 0 1 143 176 86 195 +751 4 2 0 1 70 77 124 192 +752 4 2 0 1 137 88 92 186 +753 4 2 0 1 16 151 8 196 +754 4 2 0 1 73 122 156 176 +755 4 2 0 1 3 2 49 174 +756 4 2 0 1 150 80 118 198 +757 4 2 0 1 155 117 187 195 +758 4 2 0 1 160 143 104 176 +759 4 2 0 1 84 55 59 181 +760 4 2 0 1 25 61 73 198 +761 4 2 0 1 33 30 43 190 +762 4 2 0 1 166 183 172 193 +763 4 2 0 1 105 168 2 197 +764 4 2 0 1 107 85 95 177 +765 4 2 0 1 150 117 99 195 +766 4 2 0 1 79 78 70 193 +767 4 2 0 1 141 16 20 194 +768 4 2 0 1 16 141 48 175 +769 4 2 0 1 111 180 147 191 +770 4 2 0 1 45 129 151 175 +771 4 2 0 1 86 195 176 198 +772 4 2 0 1 156 122 123 176 +773 4 2 0 1 122 86 176 198 +774 4 2 0 1 11 178 12 194 +775 4 2 0 1 29 141 20 194 +776 4 2 0 1 92 108 149 197 +777 4 2 0 1 156 176 123 192 +778 4 2 0 1 82 22 56 187 +779 4 2 0 1 160 104 123 176 +780 4 2 0 1 95 85 49 177 +781 4 2 0 1 48 40 45 175 +782 4 2 0 1 84 59 136 181 +783 4 2 0 1 51 37 42 180 +784 4 2 0 1 105 159 49 168 +785 4 2 0 1 13 187 117 196 +786 4 2 0 1 43 179 52 190 +787 4 2 0 1 139 174 189 197 +788 4 2 0 1 134 109 131 183 +789 4 2 0 1 160 86 143 176 +790 4 2 0 1 96 1 25 178 +791 4 2 0 1 114 111 147 191 +792 4 2 0 1 59 55 58 181 +793 4 2 0 1 19 26 3 174 +794 4 2 0 1 99 195 86 198 +795 4 2 0 1 110 115 82 195 +796 4 2 0 1 155 187 82 195 +797 4 2 0 1 60 62 59 181 +798 4 2 0 1 71 74 72 184 +799 4 2 0 1 25 178 15 198 +800 4 2 0 1 158 103 119 185 +801 4 2 0 1 152 140 46 189 +802 4 2 0 1 175 194 164 196 +803 4 2 0 1 43 44 52 179 +804 4 2 0 1 164 172 187 195 +805 4 2 0 1 59 62 136 181 +806 4 2 0 1 100 157 127 192 +807 4 2 0 1 93 104 143 193 +808 4 2 0 1 15 25 11 178 +809 4 2 0 1 59 58 60 181 +810 4 2 0 1 73 61 122 198 +811 4 2 0 1 99 86 80 198 +812 4 2 0 1 54 51 42 180 +813 4 2 0 1 102 119 103 185 +814 4 2 0 1 67 71 66 184 +815 4 2 0 1 71 72 66 184 +816 4 2 0 1 39 33 190 191 +817 4 2 0 1 64 63 65 182 +818 4 2 0 1 152 139 26 189 +819 4 2 0 1 160 123 122 176 +820 4 2 0 1 39 38 33 191 +821 4 2 0 1 132 88 133 186 +822 4 2 0 1 21 187 13 196 +823 4 2 0 1 92 186 87 197 +824 4 2 0 1 124 79 70 193 +825 4 2 0 1 2 26 139 174 +826 4 2 0 1 76 109 138 183 +827 4 2 0 1 114 39 33 190 +828 4 2 0 1 38 47 48 191 +829 4 2 0 1 117 13 142 187 +830 4 2 0 1 142 155 117 187 +831 4 2 0 1 153 152 46 189 +832 4 2 0 1 25 1 11 178 +833 4 2 0 1 41 147 42 180 +834 4 2 0 1 42 111 54 180 +835 4 2 0 1 98 13 117 196 +836 4 2 0 1 122 61 86 198 +837 4 2 0 1 105 83 128 168 +838 4 2 0 1 176 192 166 193 +839 4 2 0 1 107 112 85 177 +840 4 2 0 1 122 86 160 176 +841 4 2 0 1 41 147 180 191 +842 4 2 0 1 76 74 71 184 +843 4 2 0 1 13 21 142 187 +844 4 2 0 1 2 174 139 197 +845 4 2 0 1 131 71 67 184 +846 4 2 0 1 85 146 14 177 +847 4 2 0 1 142 82 155 187 +848 4 2 0 1 36 130 50 189 +849 4 2 0 1 97 87 83 197 +850 4 2 0 1 87 92 88 186 +851 4 2 0 1 41 125 47 191 +852 4 2 0 1 42 147 111 180 +853 4 2 0 1 83 168 105 197 +854 4 2 0 1 155 99 117 195 +855 4 2 0 1 155 82 115 195 +856 4 2 0 1 45 151 16 175 +857 4 2 0 1 57 102 103 185 +858 4 2 0 1 49 85 14 177 +859 4 2 0 1 158 133 103 185 +860 4 2 0 1 39 190 114 191 +861 4 2 0 1 44 144 116 179 +862 4 2 0 1 115 93 143 195 +863 4 2 0 1 8 98 9 196 +864 4 2 0 1 1 144 11 178 +865 4 2 0 1 63 100 127 182 +866 4 2 0 1 168 186 173 197 +867 4 2 0 1 146 85 112 177 +868 4 2 0 1 139 189 153 197 +869 4 2 0 1 108 153 149 197 +870 4 2 0 1 52 44 53 179 +871 4 2 0 1 32 28 31 188 +872 4 2 0 1 44 116 53 179 +873 4 2 0 1 108 92 87 197 +874 4 2 0 1 21 22 142 187 +875 4 2 0 1 176 195 164 198 +876 4 2 0 1 90 106 107 183 +877 4 2 0 1 10 11 12 194 +878 4 2 0 1 138 107 95 183 +879 4 2 0 1 82 142 22 187 +880 4 2 0 1 28 27 17 188 +881 4 2 0 1 7 21 13 196 +882 4 2 0 1 131 76 71 184 +883 4 2 0 1 100 101 157 192 +884 4 2 0 1 176 193 172 195 +885 4 2 0 1 20 16 9 194 +886 4 2 0 1 152 26 19 174 +887 4 2 0 1 88 132 75 186 +888 4 2 0 1 164 194 178 198 +889 4 2 0 1 129 31 17 188 +890 4 2 0 1 57 103 137 185 +891 4 2 0 1 127 65 63 182 +892 4 2 0 1 159 105 128 168 +893 4 2 0 1 139 152 153 189 +894 4 2 0 1 138 109 107 183 +895 4 2 0 1 137 103 133 185 +896 4 2 0 1 79 89 78 193 +897 4 2 0 1 94 43 52 190 +898 4 2 0 1 16 48 45 175 +899 4 2 0 1 25 15 61 198 +900 4 2 0 1 36 46 34 189 +901 4 2 0 1 12 29 20 194 +902 4 2 0 1 75 87 88 186 +903 4 2 0 1 77 127 157 192 +904 4 2 0 1 9 118 10 194 +905 4 2 0 1 111 114 94 190 +906 4 2 0 1 97 83 105 197 +907 4 2 0 1 99 143 86 195 +908 4 2 0 1 80 15 118 198 +909 4 2 0 1 8 151 7 196 +910 4 2 0 1 124 104 79 193 +911 4 2 0 1 134 90 109 183 +912 4 2 0 1 39 114 147 191 +913 4 2 0 1 33 43 114 190 +914 4 2 0 1 79 93 89 193 +915 4 2 0 1 77 123 124 192 +916 4 2 0 1 36 50 46 189 +917 4 2 0 1 79 104 93 193 +918 4 2 0 1 31 28 17 188 +919 4 2 0 1 97 105 139 197 +920 4 2 0 1 61 80 86 198 +921 4 2 0 1 139 105 2 197 +922 4 2 0 1 109 90 107 183 +923 4 2 0 1 8 7 13 196 +924 4 2 0 1 143 99 115 195 +925 4 2 0 1 98 8 13 196 +926 4 2 0 1 47 38 39 191 +927 4 2 0 1 41 47 147 191 +928 4 2 0 1 153 46 50 189 +929 4 2 0 1 157 101 123 192 +930 4 2 0 1 94 114 43 190 +931 4 2 0 1 115 99 155 195 +932 4 2 0 1 157 123 77 192 +933 4 2 0 1 12 20 10 194 +934 4 2 0 1 156 123 101 192 +935 4 2 0 1 61 15 80 198 +936 4 2 0 1 20 9 10 194 +937 4 2 0 1 47 39 147 191 +938 4 2 0 1 97 108 87 197 +939 4 2 0 1 139 153 108 197 +940 4 2 0 1 139 108 97 197 +941 4 2 0 1 181 158 81 136 +942 4 2 0 1 181 81 158 163 +943 4 2 0 1 69 183 78 134 +944 4 2 0 1 78 183 69 166 +945 4 2 0 1 185 161 186 163 +946 4 2 0 1 185 186 161 133 +947 4 2 0 1 180 35 189 37 +948 4 2 0 1 180 189 35 165 +949 4 2 0 1 179 121 190 167 +950 4 2 0 1 179 190 121 52 +951 4 2 0 1 177 3 4 14 +952 4 2 0 1 177 4 3 174 +953 4 2 0 1 55 181 53 58 +954 4 2 0 1 55 53 181 179 +955 4 2 0 1 145 53 181 58 +956 4 2 0 1 145 181 53 179 +957 4 2 0 1 100 192 91 101 +958 4 2 0 1 100 91 192 170 +959 4 2 0 1 154 91 192 101 +960 4 2 0 1 154 192 91 170 +961 4 2 0 1 196 129 151 7 +962 4 2 0 1 196 151 129 175 +963 4 2 0 1 189 149 130 50 +964 4 2 0 1 189 130 149 173 +965 4 2 0 1 116 178 144 1 +966 4 2 0 1 116 144 178 179 +$EndElements diff --git a/test/parallel/partition/main.cc b/test/parallel/partition/main.cc new file mode 100644 index 0000000000..a7f315cad7 --- /dev/null +++ b/test/parallel/partition/main.cc @@ -0,0 +1,193 @@ +// -*- mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- +// vi: set et ts=4 sw=4 sts=4: +/***************************************************************************** + * See the file COPYING for full copying permissions. * + * * + * This program is free software: you can redistribute it and/or modify * + * it under the terms of the GNU General Public License as published by * + * the Free Software Foundation, either version 3 of the License, or * + * (at your option) any later version. * + * * + * This program is distributed in the hope that it will be useful, * + * but WITHOUT ANY WARRANTY; without even the implied warranty of * + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * + * GNU General Public License for more details. * + * * + * You should have received a copy of the GNU General Public License * + * along with this program. If not, see . * + *****************************************************************************/ + +#include + +#include +#include +#include + +#include + +#include +#include +#include + +extern "C" { +#include +#include +} // end extern C + +template +void writeOutput(const std::string& name, const GV& gridView, int myRank) +{ + Dune::VTKWriter writer(gridView); + std::vector rank(gridView.size(0)); // resizes to number of elements (codim-0-entities) on this process only + for (const auto& element : elements(gridView, Dune::Partitions::interior)) + { + const auto eIdx = gridView.indexSet().index(element); + rank[eIdx] = myRank; + } + writer.addCellData(rank, "rank"); + writer.write(name); +} + +int main (int argc , char **argv) +{ + // if terminal output is desired, e.g. for debugging, + // set to true, else false + bool printOutput = true; + + // use MPI helper to initialize MPI + // + // This will be needed in order to set up the parallelization and dgraph structure. + // SCOTCH's dgraphs work analogously to the regular graph, but need some more inputs. + // + const auto& mpiHelper = Dune::MPIHelper::instance(argc, argv); + + // TODO add if rank == 0 around everything (read, graph, ...) + // but consider that load balance is outside of this if, + // consequently the input params need to be declared outside, too + + // read mesh file + constexpr int dim = 2; // 3 for ball.msh example + using Grid = Dune::UGGrid; + auto grid = std::shared_ptr(Dune::GmshReader::read("rectangle.msh")); + const auto& gridView = grid->leafGridView(); + writeOutput("before_loadbalance", gridView, mpiHelper.rank()); + + // create partition array (target rank for each node in graph) + std::vector scotchPartitions; + if (mpiHelper.rank() == 0) + { + using Graph = std::vector>; + + // create graph's nodes and edges from mesh + // (cell-centered methods) + constexpr int codimension = 0; + Graph graph(gridView.size(codimension)); + + // Displaying element indices + if (printOutput) + std::cout << "Element index:" << std::endl; + + for (const auto& element : elements(gridView)) + { + const auto eIdx = gridView.indexSet().index(element); + if (printOutput) + std::cout << eIdx << " "; + + for (const auto& intersection : intersections(gridView, element)) + if (intersection.neighbor()) + graph[eIdx].push_back( gridView.indexSet().index(intersection.outside())); + } + + if (printOutput) + std::cout << std::endl; + + // Number of local graph vertices (cells) + const SCOTCH_Num numNodes = graph.size(); + + // Data structures for graph input to SCOTCH (add 1 for case that + // graph size is zero) + std::vector vertTab; + vertTab.reserve(numNodes + 1); + std::vector edgeTab; + edgeTab.reserve(20*numNodes); + + // Build local graph input for SCOTCH + // (graph vertices (cells) and + // number of edges connecting two vertices) + SCOTCH_Num numEdges = 0; + vertTab.push_back(0); + for (auto vertex = graph.begin(); vertex != graph.end(); ++vertex) + { + numEdges += vertex->size(); + vertTab.push_back(vertTab.back() + vertex->size()); + edgeTab.insert(edgeTab.end(), vertex->begin(), vertex->end()); + } + + // Shrink vectors to hopefully recover any unused memory + vertTab.shrink_to_fit(); + edgeTab.shrink_to_fit(); + + // initialize graph + SCOTCH_Graph scotchGraph; + if (SCOTCH_graphInit(&scotchGraph) != 0) + DUNE_THROW(Dune::Exception, "Error initializing SCOTCH graph!"); + + // check graph's consistency (recommended before building) + if (SCOTCH_graphCheck(&scotchGraph) != 0) + DUNE_THROW(Dune::Exception, "Error within SCOTCH graph's consistency!"); + + // build graph + const SCOTCH_Num baseValue = 0; + if (SCOTCH_graphBuild(&scotchGraph, baseValue, numNodes, vertTab.data(), vertTab.data()+1, NULL, NULL, numEdges, edgeTab.data(), NULL)) + DUNE_THROW(Dune::Exception, "Error building SCOTCH graph!"); + + // initialize strategy + SCOTCH_Strat scotchStrat; + if (SCOTCH_stratInit(&scotchStrat) != 0) + DUNE_THROW(Dune::Exception, "Error initializing SCOTCH strategy!"); + + // build strategy structure + // there is also a flag for a "focus" on load balancing called SCOTCH_STRATBALANCE + // numbers from scotch tests in repo (just to test, if this entire setup works) + const SCOTCH_Num flagValue = SCOTCH_STRATDEFAULT; + const SCOTCH_Num numProcessors = mpiHelper.size(); + const double imbalanceRatio = 0.0; + + if (SCOTCH_stratGraphMapBuild(&scotchStrat, flagValue, numProcessors, imbalanceRatio) != 0) + DUNE_THROW(Dune::Exception, "Error building SCOTCH strategy!"); + + // architecture and graphMap are created and called within graphPart + // if specific ones are desired, one has to call them separately and delete the graphPart function call + // compute partitioning + scotchPartitions.resize(numNodes); + if (SCOTCH_graphPart(&scotchGraph, numProcessors, &scotchStrat, scotchPartitions.data()) != 0) + DUNE_THROW(Dune::Exception, "Error computing SCOTCH graph mapping!"); + + // free memory + SCOTCH_graphExit(&scotchGraph); + SCOTCH_stratExit(&scotchStrat); + } + + // Displaying computed graph partition array + if (printOutput && !scotchPartitions.empty()) + { + std::cout << "Graph's partition array:" << std::endl; + for (int i = 0; i < scotchPartitions.size(); ++i) + std::cout << scotchPartitions[i] << " "; + + std::cout << std::endl; + } + + // convert number types + std::vector partitions( + scotchPartitions.begin(), scotchPartitions.end() + ); + + // use UG interface to send element to target ranks + grid->loadBalance(partitions, /*grid level=*/0); + + // write grid out with ranks + writeOutput("after_loadbalance", gridView, mpiHelper.rank()); + + return 0; +} diff --git a/test/parallel/partition/rectangle.msh b/test/parallel/partition/rectangle.msh new file mode 100644 index 0000000000..17fc7903af --- /dev/null +++ b/test/parallel/partition/rectangle.msh @@ -0,0 +1,53 @@ +$MeshFormat +2.2 0 8 +$EndMeshFormat +$Nodes +20 +1 0 0 0 +2 1 0 0 +3 2 0 0 +4 3 0 0 +5 4 0 0 +6 0 1 0 +7 1 1 0 +8 2 1 0 +9 3 1 0 +10 4 1 0 +11 0 2 0 +12 1 2 0 +13 2 2 0 +14 3 2 0 +15 4 2 0 +16 0 3 0 +17 1 3 0 +18 2 3 0 +19 3 3 0 +20 4 3 0 +$EndNodes +$Elements +24 +1 2 2 0 1 6 1 2 +2 2 2 0 2 7 6 2 +3 2 2 0 3 7 11 6 +4 2 2 0 4 12 11 7 +5 2 2 0 5 16 11 12 +6 2 2 0 6 16 12 17 +7 2 2 0 7 7 2 3 +8 2 2 0 8 8 7 3 +9 2 2 0 9 8 12 7 +10 2 2 0 10 13 12 8 +11 2 2 0 11 17 12 13 +12 2 2 0 12 17 13 18 +13 2 2 0 13 8 3 4 +14 2 2 0 14 9 8 4 +15 2 2 0 15 9 13 8 +16 2 2 0 16 14 13 9 +17 2 2 0 17 18 13 14 +18 2 2 0 18 18 14 19 +19 2 2 0 19 9 4 5 +20 2 2 0 20 10 9 5 +21 2 2 0 21 10 14 9 +22 2 2 0 22 15 14 10 +23 2 2 0 23 19 14 15 +24 2 2 0 24 19 15 20 +$EndElements -- GitLab From a0f1fba74a5cf46175b3d75a699b688b237aac44 Mon Sep 17 00:00:00 2001 From: Timo Koch Date: Sun, 17 Apr 2022 21:09:42 +0200 Subject: [PATCH 2/4] [scotchbackend] Add wrapper around graph to make it reusable --- dumux/linear/scotchbackend.hh | 188 +++++++++++++++++++++------------- 1 file changed, 118 insertions(+), 70 deletions(-) diff --git a/dumux/linear/scotchbackend.hh b/dumux/linear/scotchbackend.hh index 1d8ee92832..2f653b5c2a 100644 --- a/dumux/linear/scotchbackend.hh +++ b/dumux/linear/scotchbackend.hh @@ -30,6 +30,8 @@ #include #include +#include + #if HAVE_PTSCOTCH #if HAVE_MPI #include @@ -45,6 +47,110 @@ extern "C" namespace Dumux { +#if HAVE_PTSCOTCH + +/*! + * \file + * \ingroup Linear + * \brief A wrapper around a SCOTCH graph object + */ +template +class ScotchGraph +{ +public: + //! a graph represented by an adjacency list + using Graph = std::vector>; + + ScotchGraph(const Graph& graph) + { + // Number of local graph vertices + const SCOTCH_Num numNodes = graph.size(); + + // Data structures for graph input to SCOTCH + // add 1 for case that graph size is zero + vertTab_.reserve(numNodes + 1); + edgeTab_.reserve(20*numNodes); + + // Build local graph input for SCOTCH + // (number of graph vertices (cells), + // number of edges connecting two vertices) + SCOTCH_Num numEdges = 0; + vertTab_.push_back(0); + for (auto vertex = graph.begin(); vertex != graph.end(); ++vertex) + { + numEdges += vertex->size(); + vertTab_.push_back(vertTab_.back() + vertex->size()); + edgeTab_.insert(edgeTab_.end(), vertex->begin(), vertex->end()); + } + + // Shrink vectors to hopefully recover any unused memory + vertTab_.shrink_to_fit(); + edgeTab_.shrink_to_fit(); + + if (SCOTCH_graphInit(&scotchGraph_) != 0) + DUNE_THROW(Dune::Exception, "Error initializing SCOTCH graph!"); + + // check graph's consistency (recommended before building) + if (SCOTCH_graphCheck(&scotchGraph_) != 0) + DUNE_THROW(Dune::Exception, "Error within SCOTCH graph's consistency!"); + + // build graph + const SCOTCH_Num baseValue = 0; // C-style array indexing + if (SCOTCH_graphBuild(&scotchGraph_, baseValue, numNodes, vertTab_.data(), vertTab_.data()+1, NULL, NULL, numEdges, edgeTab_.data(), NULL)) + DUNE_THROW(Dune::Exception, "Error building SCOTCH graph!"); + } + + //! Clean-up the graph + ~ScotchGraph() + { + SCOTCH_graphExit(&scotchGraph_); + } + + //! Get the raw point to the data (to pass to C interface) + SCOTCH_Graph* data() + { return &scotchGraph_; } + +private: + SCOTCH_Graph scotchGraph_; + // we have to maintain these ourselves to keep the Scotch graph valid + std::vector vertTab_; + std::vector edgeTab_; +}; + +/*! + * \file + * \ingroup Linear + * \brief A wrapper around a SCOTCH strategy object + */ +class ScotchGraphOrderStrategy +{ +public: + ScotchGraphOrderStrategy(const std::string& strategy = "") + { + if (SCOTCH_stratInit(&strategy_) != 0) + DUNE_THROW(Dune::Exception, "Error initializing SCOTCH strategy!"); + + // Set SCOTCH strategy (if provided) + if (!strategy.empty()) + SCOTCH_stratGraphOrder(&strategy_, strategy.c_str()); + } + + //! Clean-up the strategy + ~ScotchGraphOrderStrategy() + { + SCOTCH_stratExit(&strategy_); + } + + //! Get the raw point to the data (to pass to C interface) + SCOTCH_Strat* data() + { return &strategy_; } + +private: + SCOTCH_Strat strategy_; +}; + +#endif // HAVE_PTSCOTCH + /*! * \file * \ingroup Linear @@ -84,85 +190,27 @@ public: std::string scotchStrategy = "") { #if HAVE_PTSCOTCH - // Number of local graph vertices (cells) - const SCOTCH_Num vertnbr = graph.size(); - - // Data structures for graph input to SCOTCH (add 1 for case that - // graph size is zero) - std::vector verttab; - verttab.reserve(vertnbr + 1); - std::vector edgetab; - edgetab.reserve(20*vertnbr); - - // Build local graph input for SCOTCH - // (number of graph vertices (cells), - // number of edges connecting two vertices) - SCOTCH_Num edgenbr = 0; - verttab.push_back(0); - typename Graph::const_iterator vertex; - for (vertex = graph.begin(); vertex != graph.end(); ++vertex) - { - edgenbr += vertex->size(); - verttab.push_back(verttab.back() + vertex->size()); - edgetab.insert(edgetab.end(), vertex->begin(), vertex->end()); - } - - // Shrink vectors to hopefully recover an unused memory - verttab.shrink_to_fit(); - edgetab.shrink_to_fit(); - - // Create SCOTCH graph - SCOTCH_Graph scotchGraph; - - // C-style array indexing - const SCOTCH_Num baseval = 0; - - // Create SCOTCH graph and initialise - if (SCOTCH_graphInit(&scotchGraph) != 0) - { - std::cerr << "Error initializing SCOTCH graph!" << std::endl; - exit(1); - } - - // Build SCOTCH graph - if (SCOTCH_graphBuild(&scotchGraph, baseval, - vertnbr, &verttab[0], &verttab[1], NULL, NULL, - edgenbr, &edgetab[0], NULL)) - { - std::cerr << "Error building SCOTCH graph!" << std::endl; - exit(1); - } - - // Re-ordering strategy - SCOTCH_Strat strat; - SCOTCH_stratInit(&strat); - - // Set SCOTCH strategy (if provided) - if (!scotchStrategy.empty()) - SCOTCH_stratGraphOrder(&strat, scotchStrategy.c_str()); + ScotchGraph scotchGraph(graph); + ScotchGraphOrderStrategy strategy(scotchStrategy); // Vector to hold permutation vectors - std::vector permutationIndices(vertnbr); - std::vector inversePermutationIndices(vertnbr); + const auto graphSize = graph.size(); + std::vector permutationIndices(graphSize); + std::vector inversePermutationIndices(graphSize); // Reset SCOTCH random number generator to produce deterministic partitions SCOTCH_randomReset(); // Compute re-ordering - if (SCOTCH_graphOrder(&scotchGraph, &strat, permutationIndices.data(), - inversePermutationIndices.data(), NULL, NULL, NULL)) - { - std::cerr << "SCOTCH: Error during reordering graph!" << std::endl; - exit(1); - } - - // Clean up SCOTCH objects - SCOTCH_graphExit(&scotchGraph); - SCOTCH_stratExit(&strat); + if (SCOTCH_graphOrder( + scotchGraph.data(), strategy.data(), permutationIndices.data(), + inversePermutationIndices.data(), NULL, NULL, NULL + )) + DUNE_THROW(Dune::Exception, "Error reordering SCOTCH graph!"); // Copy permutation vectors - permutation.resize(vertnbr); - inversePermutation.resize(vertnbr); + permutation.resize(graphSize); + inversePermutation.resize(graphSize); std::copy(permutationIndices.begin(), permutationIndices.end(), permutation.begin()); std::copy(inversePermutationIndices.begin(), inversePermutationIndices.end(), -- GitLab From c38261c97c099f688ee3f5079a2c31830f70da75 Mon Sep 17 00:00:00 2001 From: Timo Koch Date: Sun, 17 Apr 2022 21:10:15 +0200 Subject: [PATCH 3/4] [scotchbackend] Add simple graph partitioner interface --- dumux/parallel/scotchpartitioner.hh | 120 ++++++++++++++++++++++++++++ 1 file changed, 120 insertions(+) create mode 100644 dumux/parallel/scotchpartitioner.hh diff --git a/dumux/parallel/scotchpartitioner.hh b/dumux/parallel/scotchpartitioner.hh new file mode 100644 index 0000000000..49eadf1aa4 --- /dev/null +++ b/dumux/parallel/scotchpartitioner.hh @@ -0,0 +1,120 @@ +// -*- mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- +// vi: set et ts=4 sw=4 sts=4: +/***************************************************************************** + * See the file COPYING for full copying permissions. * + * * + * This program is free software: you can redistribute it and/or modify * + * it under the terms of the GNU General Public License as published by * + * the Free Software Foundation, either version 3 of the License, or * + * (at your option) any later version. * + * * + * This program is distributed in the hope that it will be useful, * + * but WITHOUT ANY WARRANTY; without even the implied warranty of * + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * + * GNU General Public License for more details. * + * * + * You should have received a copy of the GNU General Public License * + * along with this program. If not, see . * + *****************************************************************************/ + +/*! + * \file + * \ingroup Linear + * \brief An interface to the Scotch library for graph partitioning + * \note You need to have PTSCOTCH installed to use this feature + */ +#ifndef DUMUX_PARALLEL_SCOTCH_PARTITIONER_HH +#define DUMUX_PARALLEL_SCOTCH_PARTITIONER_HH + +#include +#include +#include + +#include +#include + +namespace Dumux { + +#if HAVE_PTSCOTCH + +/*! + * \file + * \ingroup Parallel + * \brief A wrapper around a SCOTCH strategy object + */ +class ScotchGraphMapStrategy +{ +public: + ScotchGraphMapStrategy(std::size_t numProcessors, double imbalanceRatio = 0.0, int flag = SCOTCH_STRATDEFAULT) + { + if (SCOTCH_stratInit(&strategy_) != 0) + DUNE_THROW(Dune::Exception, "Error initializing SCOTCH strategy!"); + + if (SCOTCH_stratGraphMapBuild(&strategy_, static_cast(flag), static_cast(numProcessors), imbalanceRatio) != 0) + DUNE_THROW(Dune::Exception, "Error building SCOTCH strategy!"); + } + + //! Clean-up the strategy + ~ScotchGraphMapStrategy() + { + SCOTCH_stratExit(&strategy_); + } + + //! Get the raw point to the data (to pass to C interface) + SCOTCH_Strat* data() + { return &strategy_; } + +private: + SCOTCH_Strat strategy_; +}; + +#endif // HAVE_PTSCOTCH + +/*! + * \file + * \ingroup Parallel + * \brief A reordering backend using the scotch library + * \note You need to have PTSCOTCH installed to use this feature + */ +template +class ScotchPartitioner +{ +public: + //! the graph type + using Graph = std::vector>; + + //! Compute graph partition + static std::vector partition(const Graph& graph, std::size_t numProcessors) + { + std::vector targetProcessors; + partition(graph, numProcessors, targetProcessors); + return targetProcessors; + } + + //! Compute graph partition + static void partition(const Graph& graph, std::size_t numProcessors, + std::vector& targetProcessors) + { +#if HAVE_PTSCOTCH + ScotchGraph scotchGraph(graph); + ScotchGraphMapStrategy strategy(numProcessors); + + // architecture and graphMap are created and called within graphPart + // if specific ones are desired, one has to call them separately and delete the graphPart function call + // compute partitioning + const auto graphSize = graph.size(); + std::vector scotchPartitions(graphSize); + if (SCOTCH_graphPart(scotchGraph.data(), static_cast(numProcessors), strategy.data(), scotchPartitions.data()) != 0) + DUNE_THROW(Dune::Exception, "Error computing SCOTCH graph mapping!"); + + // convert number types + targetProcessors = std::vector( + scotchPartitions.begin(), scotchPartitions.end() + ); +#endif // HAVE_PTSCOTCH + } +}; + +} // end namespace Dumux + +#endif -- GitLab From 9bc4c74b707bacdaf9a24ef83da3463af4d85d89 Mon Sep 17 00:00:00 2001 From: Timo Koch Date: Sun, 17 Apr 2022 21:10:50 +0200 Subject: [PATCH 4/4] [test][parallel] Use graph partitioning interface --- test/parallel/partition/CMakeLists.txt | 4 +- test/parallel/partition/main.cc | 184 ++++++++----------------- 2 files changed, 59 insertions(+), 129 deletions(-) diff --git a/test/parallel/partition/CMakeLists.txt b/test/parallel/partition/CMakeLists.txt index 906451c366..57e9113fda 100644 --- a/test/parallel/partition/CMakeLists.txt +++ b/test/parallel/partition/CMakeLists.txt @@ -2,5 +2,5 @@ dune_symlink_to_source_files(FILES ball.msh rectangle.msh) dumux_add_test(NAME test_parallel_partition_scotch SOURCES main.cc CMAKE_GUARD "( dune-uggrid_FOUND AND PTSCOTCH_FOUND )" - MPI_RANKS 1 2 3 4 - TIMEOUT 300) + MPI_RANKS 1 2 4 + TIMEOUT 1200) diff --git a/test/parallel/partition/main.cc b/test/parallel/partition/main.cc index a7f315cad7..eb7d747304 100644 --- a/test/parallel/partition/main.cc +++ b/test/parallel/partition/main.cc @@ -29,10 +29,7 @@ #include #include -extern "C" { -#include -#include -} // end extern C +#include template void writeOutput(const std::string& name, const GV& gridView, int myRank) @@ -48,146 +45,79 @@ void writeOutput(const std::string& name, const GV& gridView, int myRank) writer.write(name); } -int main (int argc , char **argv) +template +typename Dumux::ScotchPartitioner::Graph +buildCCTpfaGraph(const GridView& gridView) { - // if terminal output is desired, e.g. for debugging, - // set to true, else false - bool printOutput = true; - - // use MPI helper to initialize MPI - // - // This will be needed in order to set up the parallelization and dgraph structure. - // SCOTCH's dgraphs work analogously to the regular graph, but need some more inputs. - // - const auto& mpiHelper = Dune::MPIHelper::instance(argc, argv); + constexpr int codimension = 0; + using Graph = typename Dumux::ScotchPartitioner::Graph; + Graph graph(gridView.size(codimension)); + + for (const auto& element : elements(gridView)) + { + const auto eIdx = gridView.indexSet().index(element); + for (const auto& intersection : intersections(gridView, element)) + if (intersection.neighbor()) + graph[eIdx].push_back(gridView.indexSet().index(intersection.outside())); + } - // TODO add if rank == 0 around everything (read, graph, ...) - // but consider that load balance is outside of this if, - // consequently the input params need to be declared outside, too + return graph; +} - // read mesh file - constexpr int dim = 2; // 3 for ball.msh example +void test2D(int rank, std::size_t numProcessors) +{ + constexpr int dim = 2; using Grid = Dune::UGGrid; auto grid = std::shared_ptr(Dune::GmshReader::read("rectangle.msh")); const auto& gridView = grid->leafGridView(); - writeOutput("before_loadbalance", gridView, mpiHelper.rank()); + writeOutput("before_loadbalance_2d", gridView, rank); // create partition array (target rank for each node in graph) - std::vector scotchPartitions; - if (mpiHelper.rank() == 0) + std::vector partitions; + if (rank == 0) { - using Graph = std::vector>; - - // create graph's nodes and edges from mesh - // (cell-centered methods) - constexpr int codimension = 0; - Graph graph(gridView.size(codimension)); - - // Displaying element indices - if (printOutput) - std::cout << "Element index:" << std::endl; - - for (const auto& element : elements(gridView)) - { - const auto eIdx = gridView.indexSet().index(element); - if (printOutput) - std::cout << eIdx << " "; - - for (const auto& intersection : intersections(gridView, element)) - if (intersection.neighbor()) - graph[eIdx].push_back( gridView.indexSet().index(intersection.outside())); - } - - if (printOutput) - std::cout << std::endl; - - // Number of local graph vertices (cells) - const SCOTCH_Num numNodes = graph.size(); - - // Data structures for graph input to SCOTCH (add 1 for case that - // graph size is zero) - std::vector vertTab; - vertTab.reserve(numNodes + 1); - std::vector edgeTab; - edgeTab.reserve(20*numNodes); - - // Build local graph input for SCOTCH - // (graph vertices (cells) and - // number of edges connecting two vertices) - SCOTCH_Num numEdges = 0; - vertTab.push_back(0); - for (auto vertex = graph.begin(); vertex != graph.end(); ++vertex) - { - numEdges += vertex->size(); - vertTab.push_back(vertTab.back() + vertex->size()); - edgeTab.insert(edgeTab.end(), vertex->begin(), vertex->end()); - } - - // Shrink vectors to hopefully recover any unused memory - vertTab.shrink_to_fit(); - edgeTab.shrink_to_fit(); - - // initialize graph - SCOTCH_Graph scotchGraph; - if (SCOTCH_graphInit(&scotchGraph) != 0) - DUNE_THROW(Dune::Exception, "Error initializing SCOTCH graph!"); - - // check graph's consistency (recommended before building) - if (SCOTCH_graphCheck(&scotchGraph) != 0) - DUNE_THROW(Dune::Exception, "Error within SCOTCH graph's consistency!"); - - // build graph - const SCOTCH_Num baseValue = 0; - if (SCOTCH_graphBuild(&scotchGraph, baseValue, numNodes, vertTab.data(), vertTab.data()+1, NULL, NULL, numEdges, edgeTab.data(), NULL)) - DUNE_THROW(Dune::Exception, "Error building SCOTCH graph!"); - - // initialize strategy - SCOTCH_Strat scotchStrat; - if (SCOTCH_stratInit(&scotchStrat) != 0) - DUNE_THROW(Dune::Exception, "Error initializing SCOTCH strategy!"); - - // build strategy structure - // there is also a flag for a "focus" on load balancing called SCOTCH_STRATBALANCE - // numbers from scotch tests in repo (just to test, if this entire setup works) - const SCOTCH_Num flagValue = SCOTCH_STRATDEFAULT; - const SCOTCH_Num numProcessors = mpiHelper.size(); - const double imbalanceRatio = 0.0; - - if (SCOTCH_stratGraphMapBuild(&scotchStrat, flagValue, numProcessors, imbalanceRatio) != 0) - DUNE_THROW(Dune::Exception, "Error building SCOTCH strategy!"); - - // architecture and graphMap are created and called within graphPart - // if specific ones are desired, one has to call them separately and delete the graphPart function call - // compute partitioning - scotchPartitions.resize(numNodes); - if (SCOTCH_graphPart(&scotchGraph, numProcessors, &scotchStrat, scotchPartitions.data()) != 0) - DUNE_THROW(Dune::Exception, "Error computing SCOTCH graph mapping!"); - - // free memory - SCOTCH_graphExit(&scotchGraph); - SCOTCH_stratExit(&scotchStrat); + const auto graph = buildCCTpfaGraph(gridView); + Dumux::ScotchPartitioner::partition(graph, numProcessors, partitions); + + if (numProcessors == 2 || numProcessors == 4) + if (std::count(partitions.begin(), partitions.end(), 0) != std::count(partitions.begin(), partitions.end(), 1)) + DUNE_THROW(Dune::Exception, "Unexpected unbalanced partition!"); } - // Displaying computed graph partition array - if (printOutput && !scotchPartitions.empty()) + grid->loadBalance(partitions, /*grid level=*/0); + writeOutput("after_loadbalance_2d", gridView, rank); +} + +void test3D(int rank, std::size_t numProcessors) +{ + constexpr int dim = 3; + using Grid = Dune::UGGrid; + auto grid = std::shared_ptr(Dune::GmshReader::read("ball.msh")); + const auto& gridView = grid->leafGridView(); + writeOutput("before_loadbalance_3d", gridView, rank); + + // create partition array (target rank for each node in graph) + std::vector partitions; + if (rank == 0) { - std::cout << "Graph's partition array:" << std::endl; - for (int i = 0; i < scotchPartitions.size(); ++i) - std::cout << scotchPartitions[i] << " "; + const auto graph = buildCCTpfaGraph(gridView); + Dumux::ScotchPartitioner::partition(graph, numProcessors, partitions); - std::cout << std::endl; + if (numProcessors == 2 || numProcessors == 4) + if (std::count(partitions.begin(), partitions.end(), 0) != std::count(partitions.begin(), partitions.end(), 1)) + DUNE_THROW(Dune::Exception, "Unexpected unbalanced partition!"); } - // convert number types - std::vector partitions( - scotchPartitions.begin(), scotchPartitions.end() - ); - - // use UG interface to send element to target ranks grid->loadBalance(partitions, /*grid level=*/0); + writeOutput("after_loadbalance_3d", gridView, rank); +} + +int main (int argc , char **argv) +{ + const auto& mpiHelper = Dune::MPIHelper::instance(argc, argv); - // write grid out with ranks - writeOutput("after_loadbalance", gridView, mpiHelper.rank()); + test2D(mpiHelper.rank(), mpiHelper.size()); + test3D(mpiHelper.rank(), mpiHelper.size()); return 0; } -- GitLab