利用Hadoop mapreduce 实现 线性正定方程组的并行计算方法
step1：eclispe 新建工程 导入hadoop2.7需要的包
搜hadoop2.7.6所有jar包 导入即可
step2:编写java代码 计算方程组
x+y=3
x+2y=5
工程代码结构如图
GaussSeidel.java
package com.kdlc.maxnum;
import java.util.*;
public class GaussSeidel {
/**
* 通过高斯迭代方式求解线性方程组
*/
public static void main(String[] args) {
System.out.println(getResult(2)[1]);
}
public static double[] getResult(int n) {
while(true){
//Scanner in = new Scanner(System.in);
//System.out.println("请输入矩阵大小：");
//int n = in.nextInt();
//int n=2;
double[][] a = new double[n + 1][n + 1];
double[] b = new double[n + 1];
double[] x = new double[n + 1];
double [] temp=new double[n+1];
/*System.out.println("请初始化矩阵：");
for (int i = 1; i <= n; i++) {
for (int j = 1; j <= n; j++) {
System.out.println("a[" + i + "][" + j + "]=");
a[i][j] = in.nextDouble();
}
}*/
a[1][1]=1;
a[1][2]=1;
a[2][1]=1;
a[2][2]=2;
/*for (int i = 1; i <= n; i++) {
System.out.println("b[" + i + "]=");
b[i] = in.nextDouble();
x[i]=0;
}*/
b[1]=3;
b[2]=5;
int k;
for(k=1;k<=1000;k++)//最大的迭代次数
{
for(int i=1;i<=n;i++)
{
temp[i]=x[i];//保存原先的值
}
for(int i=1;i<=n;i++)
{
double sum=0;
for(int j=1;j<=n;j++)
{
if(i==j)continue;
sum=sum+a[i][j]*x[j];
}
x[i]=(b[i]-sum)/a[i][i];
//System.out.println("x["+i+"]="+x[i]);
}
//System.out.println();
int m;
for(m=1;m<=n;m++)
{//有不满足条件的继续迭代
if(Math.abs(x[m]-temp[m])>=0.0001)
break;
}
if(m>n)break; //都满足条件
}
//System.out.println("迭代次数为： "+(k-1));
double[] result= new double[n];
for (int i = 1; i <= n; i++) {
result[i-1]=x[i];
//result+=" x[" + i + "]=" + x[i];
//System.out.println("x[" + i + "]=" + x[i]);
}
return result;
}
}
}
MyCombiner.java
package com.kdlc.maxnum;
import java.io.IOException;
import org.apache.hadoop.io.DoubleWritable;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;
public class MyCombiner extends Reducer<Text, IntWritable, Text, DoubleWritable> {
@Override
protected void reduce(Text key, Iterable<IntWritable> values,Context context)throws IOException, InterruptedException {
// TODO Auto-generated method stub
/* int temp = Integer.MIN_VALUE;
for(IntWritable value : values){
if(value.get() > temp){
temp = value.get();
}
}*/
double [] result=new double[2];
for(IntWritable value : values){
int n=value.get();
result=GaussSeidel.getResult(2);
}
//for(double temp:result) {
context.write(new Text("x1"), new DoubleWritable(result[0]));
context.write(new Text("x2"), new DoubleWritable(result[1]));
//context.write(new Text(), new DoubleWritable(temp));
//}
}
}
MyMapper.java
package com.kdlc.maxnum;
import java.io.IOException;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;
public class MyMapper extends Mapper<Object, Text, Text,IntWritable>{
@Override
protected void map(Object key, Text value,Context context)throws IOException, InterruptedException {
// TODO Auto-generated method stub
context.write(new Text("x"), new IntWritable(Integer.parseInt(value.toString())));
}
}
MyMaxNum.java
package com.kdlc.maxnum;
import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.DoubleWritable;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.compress.CompressionCodec;
import org.apache.hadoop.io.compress.GzipCodec;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.util.GenericOptionsParser;
public class MyMaxNum {
public static void main(String[] args) throws IOException, InterruptedException, ClassNotFoundException {
//对输出进行压缩
/*conf.setBoolean("mapred.output.compress", true);
conf.setClass("mapred.output.compression.codec", GzipCodec.class, CompressionCodec.class);*/
System.out.println("************************start**************************");
Configuration conf = new Configuration(); //设置MapReduce的配置
String[] otherArgs = new GenericOptionsParser(conf,args).getRemainingArgs();
if(otherArgs.length < 2){
System.out.println("Usage: MaxAndMinValue <in> [<in>...] <out>");
System.exit(2);
}
//Job job = new Job(conf,"");
Job job = Job.getInstance(conf,"Gauss");
job.setJarByClass(MyMaxNum.class);
job.setMapperClass(MyMapper.class);
job.setReducerClass(MyReducer.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(DoubleWritable.class);
job.setMapOutputKeyClass(Text.class);
job.setMapOutputValueClass(IntWritable.class);
//job.setCombinerClass(MyCombiner.class);
/*FileInputFormat.addInputPath(job, new Path("/cicro/opt/hadoop-2.7.7/data/workcount"));
FileOutputFormat.setOutputPath(job, new Path("/output"));*/
//设定输入输出路径
for (int i = 0; i < otherArgs.length-1;++i){
FileInputFormat.addInputPath(job,new Path(otherArgs[i]));
}
FileOutputFormat.setOutputPath(job, new Path(otherArgs[otherArgs.length-1]));
System.exit(job.waitForCompletion(true)?0:1);
}
}
MyReducer.java
package com.kdlc.maxnum;
import java.io.IOException;
import org.apache.hadoop.io.DoubleWritable;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;
public class MyReducer extends Reducer<Text, IntWritable, Text, DoubleWritable>{
private DoubleWritable resultCon=new DoubleWritable();
private DoubleWritable resultCon2=new DoubleWritable();
@Override
protected void reduce(Text key, Iterable<IntWritable> values,Context context)throws IOException, InterruptedException {
// TODO Auto-generated method stub
/*int temp = Integer.MIN_VALUE;
for(IntWritable value : values){
if(value.get() > temp){
temp = value.get();
}
}*/
double [] result=new double[2];
for(IntWritable value : values){
int n=value.get();
result=GaussSeidel.getResult(2);
}
resultCon.set(result[0]);
resultCon2.set(result[1]);
//for(double temp:result) {
context.write(new Text("x1"), resultCon);
context.write(new Text("x2"), resultCon2);
//context.write(new Text("x2"), new DoubleWritable().set(result[1]));
//context.write(new Text(), new DoubleWritable(temp));
//}
}
}
step3 eclipse把工程导出到jar包
step4 上传jar 到 hadoop share目录下面
step5 hadoop fs -mkdir input
step6 hadoop fs -mkdir output
新建文件 input.txt 输入矩阵维度
然后把input.txt 推到 hdfs input中
hadoop fs -put input.txt /cicro/opt/hadoop/data/input
最后运行
./hadoop jar /cicro/opt/hadoop-2.7.7/share/hadoop/mapreduce/gauss.jar /cicro/opt/hadoop-2.7.7/data/workcount/ /output
./hadoop fs -text /output/part-r-00000
方程得两个根就得到了
注意：
运行的时候 程序没有报错 然后output一直都生成part-r-00000.gz 结果最后发现是之前上传上去得jar影响了 但是两个jar得名字不同 可能是它只检测class得名字把 所以后来删除了原来的jar就正常了
然后main文件立马 如果mapper 和reduce得 输出格式如果不同那么：
需要分开设置：
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(DoubleWritable.class);
job.setMapOutputKeyClass(Text.class);
job.setMapOutputValueClass(IntWritable.class);
另外参考了网络资源 感谢这位作者