最近公司有這樣的業(yè)務(wù)場景, 我們有一個人工智能算法平臺系統(tǒng), 用戶可以選擇自己需要的模型, 利用我們的gpu資源來訓(xùn)練, 訓(xùn)練的過程中, 日志就是非常重要的一環(huán)了, 需要把TensorFlow或者kares訓(xùn)練的日志實(shí)時的展示到前端界面上, 用于讓用戶感知到訓(xùn)練過程.

大致的思路是前端跟java的服務(wù)器采用websocket通信, 后臺檢測到日志有變化的時候, 就主動把增量的日志文件推送到前端顯示.


在java服務(wù)中,考慮需要對日志文件做輪詢, 就想到的spring的ThreadPoolTaskScheduler, 創(chuàng)建一個定時任務(wù)的線程池, 配置為每5秒執(zhí)行一次, 檢測文件行數(shù)是否有變化, 如果文件行數(shù)有變化則返回增量數(shù)據(jù), 并記錄下目前已讀取的行數(shù).

service 代碼如下:

@Component
public class TaskService {

private final ThreadPoolTaskScheduler taskScheduler;

private final SimpMessagingTemplate messagingTemplate;

private ScheduledFuture<?> future;

private Map futureMap = new HashMap();
public TaskService(
    ThreadPoolTaskScheduler taskScheduler,
    SimpMessagingTemplate messagingTemplate) {
    this.taskScheduler = taskScheduler;
    this.messagingTemplate = messagingTemplate;
}


public void startReadLog(String jobPath){

    future = taskScheduler.schedule(new ReadLogThread(this, jobPath, messagingTemplate),new CronTrigger("*/5 * * * * ?"));

    futureMap.put(jobPath,future);
}


public void stopReadLog(String jobPath){
    ScheduledFuture temp = (ScheduledFuture) futureMap.get(jobPath);

    if(temp!=null){
        temp.cancel(true);
    }
}
}

thread 代碼如下:

public class ReadLogThread implements Runnable {

private final TaskService taskService;

private final String jobPath;

private String charset;

private final SimpMessagingTemplate messagingTemplate;

private static final String PREAMBLE_STR = "\u001B[8mha:";
private static final String POSTAMBLE_STR = "\u001B[0m";

public ReadLogThread(TaskService taskService, String jobPath,
    SimpMessagingTemplate messagingTemplate) {
    this.taskService = taskService;
    this.jobPath = jobPath;
    this.messagingTemplate = messagingTemplate;
}


@Override
public void run() {
    try {
        List<String> logs;
        if (TaskService.pageMap.containsKey(jobPath)) {
            long currentLine = (long) TaskService.pageMap.get(jobPath);

            logs = getLog(currentLine);

        } else {

            logs = getLog(0);

        }

        messagingTemplate.convertAndSend("/logs/" + jobPath, logs);

    } catch (IOException e) {
        e.printStackTrace();
    }

    System.out.println("DynamicTask.MyRunnable.run()沪伙，" + new Date());

}


    int lines = 0;
    long filePointer;
    final List<String> lastLines = new ArrayList<>(128);
    final List<Byte> bytes = new ArrayList<>();

    try (RandomAccessFile fileHandler = new RandomAccessFile(getLogFile(), "r")) {
        long fileLength = fileHandler.length() - 1;

        for (filePointer = fileLength; filePointer >maxLines;filePointer--) {
            fileHandler.seek(filePointer);
            byte readByte = fileHandler.readByte();

            if (readByte == 0x0A) {
                if (filePointer < fileLength) {
                    lines = lines + 1;
                    lastLines.add(convertBytesToString(bytes));
                    bytes.clear();
                }
            } else if (readByte != 0xD) {
                bytes.add(readByte);
            }
        }
        TaskService.pageMap.put(jobPath,fileLength);
    }

    if (lines != maxLines) {
        lastLines.add(convertBytesToString(bytes));
    }


    Collections.reverse(lastLines);

  

    return removeNotes(lastLines);
}

public File getLogFile() throws IOException {
    File rawF = new File(getRootDir(), "training.log");
    if (rawF.isFile()) {
        return rawF;
    }
    File gzF = new File(getRootDir(), "log.gz");
    if (gzF.isFile()) {
        return gzF;
    }
    //If both fail, return the standard, uncompressed log file
    return rawF;
}

public File getRootDir() throws IOException {
    return new File(getJobPath());
}

private String getJobPath() throws IOException {
    return new String(new sun.misc.BASE64Decoder().decodeBuffer(jobPath));
}

public static String humanReadableByteSize(long size) {
    String measure = "B";
    if (size < 1024) {
        return size + " " + measure;
    }
    Double number = new Double(size);
    if (number >= 1024) {
        number = number / 1024;
        measure = "KB";
        if (number >= 1024) {
            number = number / 1024;
            measure = "MB";
            if (number >= 1024) {
                number = number / 1024;
                measure = "GB";
            }
        }
    }
    DecimalFormat format = new DecimalFormat("#0.00");
    return format.format(number) + " " + measure;
}

private String convertBytesToString(List<Byte> bytes) {
    Collections.reverse(bytes);
    Byte[] byteArray = bytes.toArray(new Byte[bytes.size()]);
    return new String(ArrayUtils.toPrimitive(byteArray), getCharset());
}


public final Charset getCharset() {
    if (charset == null) {
        return Charset.defaultCharset();
    }
    return Charset.forName(charset);
}

public static List<String> removeNotes(Collection<String> logLines) {
    List<String> r = new ArrayList<String>(logLines.size());
    for (String l : logLines) {
        r.add(removeNotes(l));
    }
    return r;
}

/**
 * Removes the embedded console notes in the given log line.
 *
 * @since 1.350
 */
public static String removeNotes(String line) {
    while (true) {
        int idx = line.indexOf(PREAMBLE_STR);
        if (idx < 0) {
            return line;
        }
        int e = line.indexOf(POSTAMBLE_STR, idx);
        if (e < 0) {
            return line;
        }
        line = line.substring(0, idx) + line.substring(e + POSTAMBLE_STR.length());
    }
}


}