您的位置: 首页  >  文章  >  LINUX 如何实现多线程进行cp复制

LINUX 如何实现多线程进行cp复制

分类: 文章 • 2025-04-16 16:05:39
关于这个问题，意义虽然有限因为一般来说在复制文件的时候，实际的瓶颈来自于I/O，不管开启多少个线程实际上速度并不会快多少，但是为了练习多线程编程，
这里给出了一种C++代码实现的方式，代码附在最后。

实际上就是将一个文件分割为多个片段，开启多个线程进行同时复制，如果用户制定的并行大于服务器实际的CPU核数，程序会自动降级并行度为CPU核数，如果文件小于
100M则并行度始终为1。

[email protected]:/home/gaopeng/mmm# ./parcp log.log log10.log 2       
set parallel:2
Your cpu core is:4
real parallel:2
Will Create 2 Threads
140677902710528:0:174522367:3:4
140677894317824:174522368:349044736:3:4
Copy Thread:140677902710528 work 25%
Copy Thread:140677894317824 work 25%
Copy Thread:140677902710528 work 50%
Copy Thread:140677902710528 work 75%
Copy Thread:140677902710528 work 100%
Copy Thread:140677902710528 work Ok!!
Copy Thread:140677894317824 work 50%
Copy Thread:140677894317824 work 75%
Copy Thread:140677894317824 work Ok!!

复制完成后进行md5验证
[email protected]:/home/gaopeng/mmm# md5sum log.log
f64acc21f7187a865938b340b3eda198  log.log
[email protected]:/home/gaopeng/mmm# md5sum log10.log
f64acc21f7187a865938b340b3eda198  log10.log
可以看出校验是通过的

代码如下：

点击(此处)折叠或打开

  1. #include<iostream>
  2. #include <map>
  3. #include<stdint.h>
  4. #include<stdio.h>
  5. #include<string.h>
  6. #include<unistd.h>
  7. #include<sys/types.h>
  8. #include<sys/stat.h>
  9. #include <sys/sysinfo.h>
  10. #include<fcntl.h>
  11. #include<errno.h>
  12. #include <time.h>
  13. #include <stdarg.h>
  14. #include <stdlib.h>
  15. #include <pthread.h>
  16. #define MAX_BUFFER 65536
  17. using namespace std;



  21. pthread_mutex_t counter_mutex = PTHREAD_MUTEX_INITIALIZER;

  23. class thread_info
  24. {

  26.         private:
  27.                 uint64_t start_pos;
  28.                 uint64_t end_pos;
  29.                 int fdr;
  30.                 int fdw;
  31.         public:
  32.                 pthread_t t_id;
  33.                 static int do_id;

  35.         public:
  36.                 thread_info()
  37.                 {

  39.                         start_pos = 0;
  40.                         end_pos = 0;
  41.                         fdr = 0;
  42.                         fdw = 0;
  43.                         t_id = 0;
  44.                 }
  45.                 void set_start(uint64_t a)
  46.                 {

  48.                         start_pos = a;
  49.                 }
  50.                 void set_end(uint64_t a)
  51.                 {

  53.                         end_pos = a;
  54.                 }
  55.                 void set_fd(int a,int b)
  56.                 {

  58.                         fdr = a;
  59.                         fdw = b;
  60.                 }

  62.                 uint64_t get_start(void)
  63.                 {
  64.                         return start_pos;
  65.                 }
  66.                 uint64_t get_stop(void)
  67.                 {
  68.                         return end_pos;
  69.                 }
  70.                 int get_fdr(void)
  71.                 {
  72.                         return fdr;
  73.                 }
  74.                 int get_fdw(void)
  75.                 {
  76.                         return fdw;
  77.                 }

  79.                 void print(void)
  80.                 {
  81.                         cout<<start_pos<<":"<<end_pos<<":"<<t_id<<endl;
  82.                 }

  84. };

  86. int thread_info::do_id = 0;

  88. class ABS_dispatch
  89. {

  91.         public:
  92.                 ABS_dispatch()
  93.                 {

  95.                         par_thr = 0;
  96.                         max_cpu = 0;
  97.                 }
  98.                 virtual thread_info* call_thread(void) = 0;
  99.                 virtual void make_init(uint32_t t_n) = 0;
  100.                 uint32_t getcpu(void)
  101.                 {
  102.                         return get_nprocs() ;
  103.                 }
  104.                 virtual ~ABS_dispatch(){
  105.                 }
  106.         protected:
  107.                 uint32_t par_thr;
  108.                 uint32_t max_cpu;
  109. };

  111. class dispatch:public ABS_dispatch
  112. {

  114.         public:
  115.                 typedef multimap<uint64_t,uint64_t>::iterator pair_piece_iterator;
  116.                 dispatch():ABS_dispatch()
  117.         {
  118.                 file_idr = 0;
  119.                 file_idw = 0;
  120.                 file_size = 0;
  121.                 tread_arr_p = NULL;
  122.         }
  123.                 virtual thread_info* call_thread(void);
  124.                 virtual void make_init(uint32_t t_n)
  125.                 {
  126.                         max_cpu = getcpu(); //
  127.                         cout<<"Your cpu core is:"<<max_cpu<<"\n";
  128.                         if(t_n > max_cpu) //parallel
  129.                         {
  130.                                 cout<<"Parallel downgrad to cpu core:"<<max_cpu<<"\n";
  131.                                 par_thr = max_cpu;
  132.                         }
  133.                         else
  134.                         {
  135.                                 par_thr = t_n;
  136.                         }
  137.                 }
  138.                 void set_init(int file_idr,int file_idw)
  139.                 {

  141.                         file_size = lseek(file_idr,0,SEEK_END);
  142.                         if(file_size<100000000)
  143.                         {
  144.                                 cout<<"File small than 100M par = 1" <<"\n";
  145.                                 par_thr = 1;
  146.                         }
  147.                         this->file_idr = file_idr;
  148.                         this->file_idw = file_idw;
  149.                 }
  150.                 uint32_t real_par()
  151.                 {
  152.                         return par_thr;
  153.                 }
  154.                 virtual ~dispatch()
  155.                 {
  156.                         pair_piece.clear();
  157.                         delete [] tread_arr_p;
  158.                 }
  159.         private:
  160.                 int file_idr;
  161.                 int file_idw;
  162.                 multimap<uint64_t,uint64_t> pair_piece;
  163.                 uint64_t file_size;
  164.         public:
  165.                 thread_info* tread_arr_p;
  166. };




  171. static void* do_work(void* argc)
  172. {
  173.         uint64_t b;
  174.         uint64_t e;
  175.         int fdr;
  176.         int fdw;
  177.         char* buffer[MAX_BUFFER]={0};
  178.         thread_info* tread_arr_p;
  179.         uint64_t loopc = 0;
  180.         uint64_t loopc25 = 0;
  181.         uint64_t i = 0;
  182.         int m = 1;
  183.         pthread_t t_id;
  184.         tread_arr_p = static_cast<thread_info*>(argc);

  186.         //临界区 MUTEX
  187.         pthread_mutex_lock(&counter_mutex);
  188.         b = (tread_arr_p+ tread_arr_p->do_id)->get_start();
  189.         e = (tread_arr_p+ tread_arr_p->do_id)->get_stop();
  190.         fdr = (tread_arr_p+ tread_arr_p->do_id)->get_fdr();
  191.         fdw = (tread_arr_p+ tread_arr_p->do_id)->get_fdw();
  192.         t_id = (tread_arr_p+ tread_arr_p->do_id)->t_id ;
  193.         cout<< t_id <<":"<<b<<":"<<e<<":"<<fdr<<":"<<fdw<<"\n";
  194.         tread_arr_p->do_id++;
  195.         pthread_mutex_unlock(&counter_mutex);
  196.         //临界区
  197.         loopc = e/uint64_t(MAX_BUFFER);
  198.         loopc25 = loopc/(uint64_t)4;

  200.         while(i<loopc)
  201.         {

  203.                 if(i == loopc25*m )
  204.                 {
  205.                         cout<< "Copy Thread:"<<t_id<<" work "<<25*m<<"%\n";
  206.                         m++;
  207.                 }

  209.                 memset(buffer,0,MAX_BUFFER);
  210.                 pread(fdr,buffer,MAX_BUFFER,uint64_t(i*MAX_BUFFER));
  211.                 pwrite(fdw,buffer,MAX_BUFFER,uint64_t(i*MAX_BUFFER));
  212.                 i++;
  213.         }
  214.         memset(buffer,0,MAX_BUFFER);
  215.         pread(fdr,buffer,(e-uint64_t(i*MAX_BUFFER)),uint64_t(i*MAX_BUFFER));
  216.         pwrite(fdw,buffer,(e-uint64_t(i*MAX_BUFFER)),uint64_t(i*MAX_BUFFER));
  217.         cout<< "Copy Thread:"<<t_id<<" work Ok!!"<<"\n";
  218.         return NULL;
  219. }

  221. thread_info* dispatch::call_thread()
  222. {

  224.         int i = 0;
  225.         uint64_t temp_size = 0;
  226.         temp_size = file_size/par_thr;
  227.         tread_arr_p = new thread_info[par_thr];

  229.         cout<<"Will Create "<<par_thr<<" Threads\n";
  230.         //cout<<tread_arr_p<<endl;
  231.         //cout<<sizeof(thread_info)<<endl;
  232.         for(i = 0;i<par_thr-1;i++)
  233.         {
  234.                 pair_piece.insert( pair<uint64_t,uint64_t>(temp_size*i,temp_size*(i+1)-1 ));
  235.         }
  236.         pair_piece.insert( pair<uint64_t,uint64_t>(temp_size*i,file_size ));
  237.         i = 1;
  238.         for(pair_piece_iterator it =pair_piece.begin();it !=pair_piece.end() ;it++)
  239.         {
  240.                 //cout<<"--Thread: "<<i<<"\n";
  241.                 //cout<<it->first<<"\n";
  242.                 //cout<<it->second<<"\n";

  244.                 //cout<<tread_arr_p+(i-1)<<endl;
  245.                 (tread_arr_p+(i-1))->set_start(it->first);
  246.                 (tread_arr_p+(i-1))->set_end(it->second);
  247.                 (tread_arr_p+(i-1))->set_fd(file_idr,file_idw);
  248.                 pthread_create(&((tread_arr_p+(i-1))->t_id),NULL,do_work,static_cast<void*>(tread_arr_p));
  249.                 //(tread_arr_p+(i-1))->print();
  250.                 i++;
  251.         }

  253.         return tread_arr_p;
  254. }



  258. int main(int argc,char** argv)
  259. {

  261.         dispatch test;
  262.         thread_info* thread_info_p = NULL;
  263.         uint32_t real_par;
  264.         void *tret;

  266.         int fdr = open(argv[1],O_RDONLY|O_NOFOLLOW);
  267.         int fdw = open(argv[2],O_RDWR|O_CREAT|O_EXCL,0755);
  268.         cout<<"Author: gaopeng QQ:22389860 Blog:http://blog.itpub.net/7728585\n";
  269.         if(argc<4)
  270.         {
  271.                 cout<<"USAGE:parcp sourcefile destfile paralle\n";
  272.                 return -1;
  273.         }
  274.         if(fdr == -1 || fdw == -1)
  275.         {
  276.                 perror("open readfile:");
  277.                 return -1;
  278.         }
  279.         if(fdw == -1)
  280.         {
  281.                 perror("open wirtefile:");
  282.                 return -1;
  283.         }

  285.         if(sscanf(argv[3],"%u",&real_par) == EOF)
  286.         {
  287.                 perror("sscanf:");
  288.                 return -1;
  289.         }

  291.         cout<<"set parallel:"<<real_par<<"\n";

  293.         //cout<<lseek(fd,0,SEEK_SET) <<endl;
  294.         test.make_init(real_par);
  295.         test.set_init(fdr,fdw);
  296.         real_par = test.real_par();
  297.         cout<<"real parallel:" <<real_par<<"\n";
  298.         thread_info_p = test.call_thread();
  299.         for(int i = 0 ;i<real_par;i++)
  300.         {
  301.                 //cout<<(thread_info_p+i)->t_id<<endl;
  302.                 pthread_join((thread_info_p+i)->t_id,&tret);
  303.                 //cout<<reinterpret_cast<long>(tret)<<endl;
  304.         }
  305.         close(fdw);
  306.         close(fdr);
  307. }
如果觉得不错 您可以考虑请作者喝杯茶(微信支付)：                                作者微信号：

LINUX 如何实现多线程进行cp复制                   LINUX 如何实现多线程进行cp复制

来自 “ ITPUB博客 ” ，链接：http://blog.itpub.net/7728585/viewspace-2137204/，如需转载，请注明出处，否则将追究法律责任。

转载于:http://blog.itpub.net/7728585/viewspace-2137204/

相关推荐