最近有個(gè)需求，需要把爬下來的數(shù)據(jù)寫入到文件中纸巷，于是乎查了下相關(guān)的博客和文檔镇草，大概記錄一下，方便以后自己用瘤旨，這個(gè)基本上都是復(fù)制粘貼來的梯啤，主要是方便自己用，其他人看到了覺得有用就用存哲，沒用也可以吐槽因宇，就是抄的反正。
Nodejs文件覆蓋祟偷、追加察滑、讀取、全量復(fù)制
同步版 從這里復(fù)制的
邏輯處理類的實(shí)現(xiàn)：syncdata.js
const fs = require("fs");
/**
* update file
* @param res to update content
* @returns update operation result
*/
async function updateResFile (res) {
return new Promise((resolve, reject) => {
fs.writeFile(`${conf['workdir']}${conf['res']}`, res, (err) => {
if (err) {
console.error(`error happened when updateResFile inner: ${err.message}`);
reject(false);
return;
} else {
resolve(true);
return;
}
});
}).catch(error => {
console.error(`error happened when updateResFile: ${error.message}`);
return false;
});
}
/**
* break line append log
* @param log log content
* @returns append operation result
*/
async function appendFile (log) {
return new Promise((resolve, reject) => {
if ("windows" == conf['os']) {
log+='\r\n';
} else {
log+='\n';
}
fs.appendFile(`${conf['workdir']}${conf['log']}`, log, (err) => {
if (err) {
console.error(`error happened when appendFile inner: ${err.message}`);
reject(false);
return;
} else {
resolve(true);
return;
}
});
}).catch(error => {
console.error(`error happened when appendFile: ${error.message}`);
});
}
/**
* execute copy file logic
* @param {*} src
* @param {*} dst
* @param {*} callback
*/
async function handleDir (src, dst) {
return new Promise(async (resolve, reject) => {
fs.access(dst, fs.constants.F_OK, async (err) => {
if (err) {
fs.mkdir(dst, { recursive: true }, async (err) => {
if (err) {
console.error(`error happened when handleDir.mkdir: ${err.message}`);
reject(`error happened when handleDir.mkdir: ${err.message}`);
return;
} else {
await copyDir(src, dst);
}
});
} else {
await copyDir(src, dst);
}
resolve(true);
});
}).catch(error => {
console.error(`error happened when handleDir: ${error.message}`);
});
}
/**
* execute iterator copy directory
* @param src directory source path
* @param dst directory destination path
* @returns copy result
*/
async function copyDir (src, dst) {
return new Promise(async (resolve, reject) => {
fs.readdir(src, (err, paths) => {
paths.forEach(function(path){
var _src=src+'/'+path;
var _dst=dst+'/'+path;
fs.stat(_src, async function(err,stats){
if(err) {
console.error(`copyDir failed, caused by ${JSON.stringify(err)}`);
reject(false);
} else if(stats.isFile()){
let readable=fs.createReadStream(_src);
let writable=fs.createWriteStream(_dst);
readable.pipe(writable);
readable.on('end', () => {
resolve(true);
});
}else if(stats.isDirectory()){
await handleDir(_src, _dst, copyDir);
}
});
});
});
}).catch(error => {
console.error(`error happened when copyDir: ${error.message}`);
});
}
/**
* read file sync
* @param path filefullpath
* @returns file content
*/
async function readFile (path) {
return new Promise((resolve, reject) => {
fs.readFile(path, conf['encode'] , (err, data) => {
if (err) {
console.error(`readFile failed, caused by ${JSON.stringify(err)}`);
reject(false);
} else {
resolve(data);
return;
}
});
}).catch(error => {
console.error(`error happened when readFile: ${error.message}`);
});
}
/**
* main entrance
*/
async function syncFile () {
console.log(`begin ------- ${new Date().getTime()}`);
let cpRes = await copyDir('D:/開發(fā)文檔/郵件文檔/1-郵件附件', './2');
console.log(`result is ${cpRes}`);
console.log(`end ------- ${new Date().getTime()}`);
}
// call
syncFile ();
同一目錄下的配置文件conf.json
{
"os" : "windows",
"encode" : "utf-8",
"workdir" : "/home/",
"latestVersion" : "latestVersion.txt",
"currentVersion" : "currentVersion.txt",
"log" : "log.txt",
"res" : "syncResult.txt",
"copySrc" : "/copySrc/",
"copyDst" : "/copyDst/",
"success-text" : "success",
"fail-text" : "fail",
"dateformat" : "YYYY-MM-DD"
}
執(zhí)行方式：
cd到syncdata.js同一級(jí)目錄修肠，執(zhí)行node syncdata.js
異步版 從這里復(fù)制的
所有方法都是異步的贺辰，如果需要同步則需要修改fs的方法，并使用Promise進(jìn)行包裝嵌施，另外需要注意饲化，全部復(fù)制的方法需要傳入兩個(gè)目錄，不帶文件的path吗伤，并且是有最大文件打開數(shù)量限制的吃靠，如果是大文件但文件數(shù)量較少則適合使用，如果是小文件且數(shù)量大足淆，則不適合巢块，建議進(jìn)行先打成壓縮包礁阁，再進(jìn)行傳輸。
邏輯處理類的實(shí)現(xiàn)：syncdata.js
const fs = require("fs");
// local path
const conf = require("./conf.json");
/**
* update file
* @param res to update content
* @returns update operation result
*/
function updateResFile (res) {
return fs.writeFileSync(`${conf['workdir']}${conf['res']}`, res);
}
/**
* break line append log
* @param log log content
* @returns append operation result
*/
function appendFile (log) {
if ("windows" == conf['os']) {
log+='\r\n';
} else {
log+='\n';
}
return fs.appendFileSync(`${conf['workdir']}${conf['log']}`, log);
}
/**
* handle directory
* @param {*} src
* @param {*} dst
* @param {*} callback
*/
function handleDir(src, dst) {
fs.access(dst, fs.constants.F_OK, (err) => {
if (err) {
fs.mkdirSync(dst);
copyDir(src, dst);
} else {
copyDir(src, dst);
}
});
}
/**
* execute iterator copy directory
* @param src directory source path
* @param dst directory destination path
* @returns copy result
*/
function copyDir (src, dst) {
// read file path sync
let paths = fs.readdirSync(src);
paths.forEach(function(path){
var _src=src+'/'+path;
var _dst=dst+'/'+path;
//stats object, include its own attribute
fs.stat(_src,function(err,stats){
if(err) {
console.log(`copyDir failed, caused by ${JSON.stringify(err)}`);
return false;
} else if(stats.isFile()){
let readable=fs.createReadStream(_src);
let writable=fs.createWriteStream(_dst);
readable.pipe(writable);
}else if(stats.isDirectory()){
handleDir(_src, _dst, copyDir);
}
});
});
return true;
}
/**
* read file sync
* @param path filefullpath
* @returns file content
*/
function readFile (path) {
return fs.readFileSync(path, conf['encode']);
}
/**
* main entrance
*/
function syncFile () {
// test code here
}
// call
syncFile ();
同一目錄下的配置文件`conf.json`
```json
{
"os" : "windows",
"encode" : "utf-8",
"workdir" : "/home/",
"latestVersion" : "latestVersion.txt",
"currentVersion" : "currentVersion.txt",
"log" : "log.txt",
"res" : "syncResult.txt",
"copySrc" : "/copySrc/",
"copyDst" : "/copyDst/",
"success-text" : "success",
"fail-text" : "fail",
"dateformat" : "YYYY-MM-DD"
}
執(zhí)行方式：
cd到syncdata.js同一級(jí)目錄夕冲，執(zhí)行node syncdata.js
