ESyS-Particle  4.0.1
LatticeMaster.hpp
00001 
00002 //                                                         //
00003 // Copyright (c) 2003-2011 by The University of Queensland //
00004 // Earth Systems Science Computational Centre (ESSCC)      //
00005 // http://www.uq.edu.au/esscc                              //
00006 //                                                         //
00007 // Primary Business: Brisbane, Queensland, Australia       //
00008 // Licensed under the Open Software License version 3.0    //
00009 // http://www.opensource.org/licenses/osl-3.0.php          //
00010 //                                                         //
00012 
00013 #include "Parallel/GetRef_cmd.h"
00014 #include <map>
00015 #include <set>
00016 
00017 template <typename TmplVisitor>
00018 void CLatticeMaster::visitMeshFaceReferences(const string &meshName)
00019 {
00020   throw std::runtime_error("CLatticeMaster::visitMeshFaceReferences: Not implemented.");
00021 }
00022 
00023 template <typename TmplVisitor>
00024 void CLatticeMaster::visitMesh2dNodeReferences(const string &meshName, TmplVisitor &visitor)
00025 {
00026   console.XDebug()<<"CLatticeMaster::visitMesh2dNodeReferences( " << meshName << ")\n";
00027   GetNodeRefCommand cmd(getGlobalRankAndComm(),meshName);
00028   // broadcast command
00029   cmd.broadcast();
00030   
00031   // receive data (multimap)
00032   std::multimap<int,int> ref_mmap;
00033   m_tml_global_comm.gather(ref_mmap);
00034   // collate into set
00035   std::set<int> ref_set; //== this is the set of node ids == 
00036   for(std::multimap<int,int>::iterator iter=ref_mmap.begin();
00037       iter!=ref_mmap.end();
00038       iter++){
00039     ref_set.insert(iter->second);
00040   }
00041   for (std::set<int>::const_iterator it = ref_set.begin(); it != ref_set.end(); it++)
00042   {
00043     visitor.visitNodeRef(*it);
00044   }
00045   console.XDebug()<<"end CLatticeMaster::visitMesh2dNodeReferences()\n";
00046 }
00047 
00048 template <typename TmplVisitor>
00049 void CLatticeMaster::visitMesh2dEdgeStress(const string &meshName, TmplVisitor &visitor)
00050 {
00051   console.XDebug()<<"CLatticeMaster::visitMesh2dEdgeStress( " << meshName << ")\n";
00052   std::multimap<int,pair<int,Vec3> > temp_mm;
00053   std::map<int,Vec3> data; //=== map of id, value ===
00054 
00055   BroadcastCommand cmd(getGlobalRankAndComm(), CMD_GETMESH2DSTRESS);
00056   cmd.append(meshName.c_str());
00057   cmd.broadcastCommand();
00058   cmd.broadcastBuffer();
00059 
00060   // get data from slaves
00061   m_tml_global_comm.gather(temp_mm);
00062   
00063   // add data together
00064   for(std::multimap<int,pair<int,Vec3> >::iterator iter=temp_mm.begin();
00065       iter!=temp_mm.end();
00066       iter++){
00067     if(data.find((iter->second).first)==data.end()){ // id not in data -> insert
00068       data.insert(iter->second);
00069     } else { // id is in data -> add
00070       data[(iter->second).first]+=(iter->second).second;
00071     }
00072   }
00073 
00074   for (std::map<int,Vec3>::const_iterator it = data.begin(); it != data.end(); it++)
00075   {
00076     visitor.visitRefStressPair(it->first, it->second);
00077   }
00078 
00079   cmd.wait("visitMesh2dEdgeStress");
00080   console.XDebug()<<"end CLatticeMaster::visitMesh2dEdgeStress()\n";  
00081 }
00082 
00083 template <typename TmplVisitor>
00084 void CLatticeMaster::visitTriMeshFaceForce(
00085   const string &meshName,
00086   TmplVisitor &visitor
00087 )
00088 {
00089   console.XDebug()<<"CLatticeMaster::visitTriMeshFaceForce( " << meshName << ")\n";
00090   std::multimap<int,pair<int,Vec3> > temp_mm;
00091   std::map<int,Vec3> data; //=== map of id, value ===
00092 
00093   BroadcastCommand cmd(getGlobalRankAndComm(), CMD_GETTRIMESHFORCE);
00094   cmd.append(meshName.c_str());
00095   cmd.broadcastCommand();
00096   cmd.broadcastBuffer();
00097 
00098   // get data from slaves
00099   m_tml_global_comm.gather(temp_mm);
00100   
00101   // add data together
00102   for(std::multimap<int,pair<int,Vec3> >::iterator iter=temp_mm.begin();
00103       iter!=temp_mm.end();
00104       iter++){
00105     if(data.find((iter->second).first)==data.end()){ // id not in data -> insert
00106       data.insert(iter->second);
00107     } else { // id is in data -> add
00108       data[(iter->second).first]+=(iter->second).second;
00109     }
00110   }
00111 
00112   for (std::map<int,Vec3>::const_iterator it = data.begin(); it != data.end(); it++)
00113   {
00114     visitor.visitRefForcePair(it->first, it->second);
00115   }
00116 
00117   cmd.wait("visitTriMeshFaceStress");
00118   console.XDebug()<<"end CLatticeMaster::visitTriMeshFaceForce()\n";  
00119 }
00120 
00121 template <typename TmplVisitor, typename TmplParticle>
00122 void CLatticeMaster::visitParticlesOfType(
00123   const IdVector &particleIdVector,
00124   TmplVisitor &visitor
00125 )
00126 {
00127   console.Debug() << "CLatticeMaster::visitParticlesOfType: enter\n";
00128   typedef std::multimap<int,TmplParticle> ParticleMMap;
00129   ParticleMMap particleMMap;
00130 
00131   console.Debug()
00132     << "CLatticeMaster::visitParticlesOfType: broadcasting command\n";
00133   BroadcastCommand cmd(getGlobalRankAndComm(), CMD_GETIDPARTICLEDATA);
00134   cmd.broadcastCommand();
00135 
00136   console.Debug()
00137     << "CLatticeMaster::visitParticlesOfType: broadcasting particle id's\n";
00138   m_tml_global_comm.broadcast_cont(particleIdVector);
00139 
00140   console.Debug()
00141     << "CLatticeMaster::visitParticlesOfType:"
00142     << " gathering particle data from workers\n";
00143   m_tml_global_comm.gather_packed(particleMMap);
00144   console.Debug()
00145     << "CLatticeMaster::visitParticlesOfType:"
00146     << " gathered " << particleMMap.size() << " particles\n";
00147 
00148   console.Debug()
00149     << "CLatticeMaster::visitParticlesOfType:"
00150     << " visiting particle data\n";
00151   for(
00152     typename ParticleMMap::iterator iter=particleMMap.begin();
00153     iter != particleMMap.end();
00154     iter++
00155   )
00156   {
00157     iter->second.visit(visitor);
00158   }
00159 
00160   cmd.wait("visitParticles");
00161   console.Debug() << "CLatticeMaster::visitParticlesOfType: exit\n";
00162 }
00163 
00164 template <typename TmplVisitor>
00165 void CLatticeMaster::visitParticles(
00166   const IdVector &particleIdVector,
00167   TmplVisitor &visitor
00168 )
00169 {
00170   console.Debug() << "CLatticeMaster::visitParticles: enter\n";
00171 
00172   if (m_particle_type == "Basic")
00173   {
00174     visitParticlesOfType<TmplVisitor,CParticle>(particleIdVector, visitor);
00175   }
00176   else if (m_particle_type == "Rot")
00177   {
00178     visitParticlesOfType<TmplVisitor,CRotParticle>(particleIdVector, visitor);
00179   }
00180   else if (m_particle_type == "RotVi")
00181   {
00182     visitParticlesOfType<TmplVisitor,CRotParticleVi>(particleIdVector, visitor);
00183   }
00184   else if (m_particle_type == "RotTherm")
00185   {
00186     visitParticlesOfType<TmplVisitor,CRotThermParticle>(particleIdVector, visitor);
00187   }
00188   else
00189   {
00190     throw
00191       std::runtime_error(
00192         std::string("Unknown particle type: ") + m_particle_type
00193       );
00194   }
00195   console.Debug() << "CLatticeMaster::visitParticles: exit\n";
00196 }
00197 
00198 template <class TmplIterator, class TmplParticle>
00199 void CLatticeMaster::addParticles(TmplIterator &particleIt)
00200 {
00201   CMPIBarrier barrier(m_global_comm);
00202   CMPILCmdBuffer cmd_buffer(m_global_comm,m_global_rank);
00203 
00204   // cast storage pointer to correct type 
00205   vector<TmplParticle> particleVector;
00206   console.XDebug()
00207     << "CLatticeMaster::addParticles:"
00208     << " Reserving vector memory for particles.\n";
00209 
00210   const int numBroadcastParticles = 50000;
00211   particleVector.reserve(numBroadcastParticles);
00212   console.XDebug()
00213     << "CLatticeMaster::addParticles:"
00214     << " Beginning add-particle loop..." << "\n";
00215   for (int i = 0; particleIt.hasNext(); i++) {
00216     const TmplParticle particle(particleIt.next());
00217 
00218     particleVector.push_back(particle);
00219     if (((i+1) % 5000) == 0) {
00220       console.XDebug()
00221         << "CLatticeMaster::addParticles:"
00222         << "Adding particle with id "
00223         << particleVector.rbegin()->getID() << "\n";
00224     }
00225 
00226     if (((i+1) % numBroadcastParticles) == 0)
00227     {
00228       console.XDebug() << "CLatticeMaster::addParticles:"
00229         << " Broadcasting receive cmd...." << "\n";
00230       cmd_buffer.broadcast(CMD_RECEIVEPARTICLES);
00231       console.XDebug()
00232         << "CLatticeMaster::addParticles: Broadcasting particles...." << "\n";
00233       m_tml_global_comm.broadcast_cont_packed(particleVector);
00234       barrier.wait("CLatticeMaster::addParticles: Post particle broadcast.");
00235       barrier.wait("CLatticeMaster::addParticles: Post Command.");
00236       particleVector.clear();
00237       particleVector.reserve(numBroadcastParticles);
00238     }
00239   }
00240   console.XDebug()
00241     << "CLatticeMaster::addParticles: Done add-particle loop..." << "\n";
00242   console.XDebug()
00243     << "CLatticeMaster::addParticles: Broadcasting final particle-receive cmd"
00244     << "\n";
00245   cmd_buffer.broadcast(CMD_RECEIVEPARTICLES);
00246   console.XDebug() << "CLatticeMaster::addParticles:"
00247     << " Broadcasting final set of particles...\n";
00248   m_tml_global_comm.broadcast_cont_packed(particleVector);
00249   barrier.wait("Post final particle broadcast.");
00250   barrier.wait("Post final particle-broadcast command.");
00251   // -- build ntable
00252   console.XDebug()
00253     << "CLatticeMaster::addParticles: "
00254     << "Building ntable (searchNeighbours)...\n";
00255   searchNeighbors(true);
00256   console.XDebug()
00257     << "CLatticeMaster::addParticles: exit\n";
00258 
00259 }
00260 
00261 template <class TmplIterator>
00262 void CLatticeMaster::addConnections(TmplIterator &connectionIt)
00263 {
00264   console.XDebug()
00265     << "CLatticeMaster::addConnections: enter\n";
00266   
00267   CMPIBarrier barrier(m_global_comm);
00268   CMPILCmdBuffer cmd_buffer(m_global_comm,m_global_rank);
00269   
00270   const int numBroadcastConnections = 100000;
00271   vector<int> connectionBuffer;
00272   connectionBuffer.reserve(numBroadcastConnections);
00273   int i = 0;
00274   for (; connectionIt.hasNext(); i++)
00275   {
00276     typename TmplIterator::value_type data = connectionIt.next();
00277     connectionBuffer.push_back(data.getTag());
00278     connectionBuffer.push_back(data.getP1Id());
00279     connectionBuffer.push_back(data.getP2Id());
00280     if ((i+1) % 50000 == 0)
00281     {
00282       console.XDebug() << "Adding connection number " << i << "\n";
00283     }
00284     if ((i+1) % numBroadcastConnections == 0)
00285     {
00286       console.XDebug() << "CLatticeMaster::addConnections:"
00287         << " Broadcasting receive cmd...." << "\n";
00288       cmd_buffer.broadcast(CMD_RECEIVECONNECTIONS);
00289       console.XDebug()
00290         << "CLatticeMaster::addConnections: Broadcasting connections...." << "\n";
00291       m_tml_global_comm.broadcast_cont_packed(connectionBuffer);
00292       barrier.wait("CLatticeMaster::addConnections: Post connection broadcast.");
00293       barrier.wait("CLatticeMaster::addConnections: Post Command.");
00294       connectionBuffer.clear();
00295       connectionBuffer.reserve(numBroadcastConnections);
00296     }
00297     //m_temp_conn[data.getTag()].push_back(data.getP1Id());
00298     //m_temp_conn[data.getTag()].push_back(data.getP2Id());
00299   }
00300   console.XDebug()
00301     << "CLatticeMaster::addConnections: Done add-connection loop..." << "\n";
00302   console.XDebug()
00303     << "CLatticeMaster::addConnections: Broadcasting final connection-receive cmd"
00304     << "\n";
00305   cmd_buffer.broadcast(CMD_RECEIVECONNECTIONS);
00306   console.XDebug() << "CLatticeMaster::addConnections:"
00307     << " Broadcasting final set of connections...\n";
00308   m_tml_global_comm.broadcast_cont_packed(connectionBuffer);
00309   barrier.wait("Post final connection broadcast.");
00310   barrier.wait("Post final connection-broadcast command.");
00311 
00312   console.XDebug()<< "Added " << i << " connections..." << "\n";
00313   // -- build ntable
00314   searchNeighbors(true);
00315   console.XDebug()
00316     << "CLatticeMaster::addConnections: exit\n";
00317 }