package es.usj.svit.simuhaff import es.usj.svit.util.IO import es.usj.svit.simuhaff.approach.domain.KromaiaFormat import es.usj.svit.simuhaff.approach.encoding.Mapper import es.usj.svit.simuhaff.approach.encoding.XMLEncodeHelper import es.usj.svit.simuhaff.approach.simulation.EAlgorithm fun main(args: Array) { println("SimuHaFF - Simulation and Human as Fitness Function") println("Program arguments: ${if (args.isNotEmpty()) args.joinToString() else "n/a"}") val bugTypes = arrayOf( "BAD_LINK_INDEXES", "HULL_MOVEMENT_BLOCKED", "HULL_NOT_LINKED", "WEAK_POINT_HIDDEN", "WEAK_POINT_OVERLAPPED" ) // -- Expert 1 ---------------- (@dblasco) // order=V1-BS-V2-V3 launchSimulations(arrayOf("V1"), arrayOf("V1"), bugTypes, sizeBugCatalog = 1, numberOfExecutions = 1) launchSimulations(arrayOf("BS"), arrayOf("BS"), bugTypes, sizeBugCatalog = 1, numberOfExecutions = 1) launchSimulations(arrayOf("V2"), arrayOf("V2"), bugTypes, sizeBugCatalog = 1, numberOfExecutions = 1) launchSimulations(arrayOf("V3"), arrayOf("V3"), bugTypes, sizeBugCatalog = 1, numberOfExecutions = 1) // -- Expert 2 ---------------- (@jchueca) // order=V2-V3-BS-V1 launchSimulations(arrayOf("V2"), arrayOf("V2"), bugTypes, sizeBugCatalog = 1, numberOfExecutions = 1) launchSimulations(arrayOf("V3"), arrayOf("V3"), bugTypes, sizeBugCatalog = 1, numberOfExecutions = 1) launchSimulations(arrayOf("BS"), arrayOf("BS"), bugTypes, sizeBugCatalog = 1, numberOfExecutions = 1) launchSimulations(arrayOf("V1"), arrayOf("V1"), bugTypes, sizeBugCatalog = 1, numberOfExecutions = 1) // -- Expert 3 ---------------- (@jveron) // order=V3-V1-V2-BS launchSimulations(arrayOf("V3"), arrayOf("V3"), bugTypes, sizeBugCatalog = 1, numberOfExecutions = 1) launchSimulations(arrayOf("V1"), arrayOf("V1"), bugTypes, sizeBugCatalog = 1, numberOfExecutions = 1) launchSimulations(arrayOf("V2"), arrayOf("V2"), bugTypes, sizeBugCatalog = 1, numberOfExecutions = 1) launchSimulations(arrayOf("BS"), arrayOf("BS"), bugTypes, sizeBugCatalog = 1, numberOfExecutions = 1) // val names = arrayOf( // "Argos", // "DaimonTrial", // "DaimonZeta", // "Orion", // "Teuthus", // "Vermis" // ) // launchSimulations(args, names, bugTypes, sizeBugCatalog = (2..3).random(), numberOfExecutions = 1) } fun launchSimulations( args: Array, names: Array, bugTypes: Array, sizeBugCatalog: Int = 0, numberOfExecutions: Int = 30 ) { val outputBasePath = "out${if (args.isNotEmpty()) "/" + args[0] else ""}" val encodings = XMLEncodeHelper.encode(models = names.map { model -> val config = IO.getConfiguration("xmlb/$model.cfg") Mapper.mapFromConfigToModels(config) }.flatten(), KromaiaFormat::class) val encodingSubset = mutableListOf() for (i in 0 until sizeBugCatalog) { encodingSubset.add(encodings.random()) } val confusionMatricesPerBugType = mutableMapOf>>>() val confusionMatrices = mutableListOf>>() encodingSubset.forEachIndexed { n, encoding -> val confusionMatrix = mutableListOf>() for (i in 0 until numberOfExecutions) { val oracle = IO.getConfiguration( "${encoding.name.substring(0, encoding.name.lastIndexOf('_'))}_oracle.txt" ).first() val simulation = EAlgorithm( encoding = encoding, name = encoding.name, oracle = oracle, randomSearch = false, meta = mapOf("modelNumber" to n + 1, "modelsTotalNumber" to sizeBugCatalog) ).run(variant = EAlgorithm.Variant.valueOf(args[0])) confusionMatrix.add(simulation.confusionMatrixRow) } confusionMatricesPerBugType.getOrPut(bugTypes.find { encoding.name.contains(it) }!!) { mutableListOf() } += confusionMatrix confusionMatrices.add(confusionMatrix) IO.writeToFile( "$outputBasePath/${encoding.name.substring(0, encoding.name.length - 4)}.csv", arrayOf("TP;FP;FN;TN") + confusionMatrix.map { it.joinToString(separator = ";") }.toTypedArray() ) } /** * Calculate confusion matrices per bug type. */ for (bugType in confusionMatricesPerBugType.keys) { val meanConfusionMatrix = mutableListOf("TP;FP;FN;TN") for (confusionMatrix in confusionMatricesPerBugType[bugType]!!) { var meanTP = 0.0 var meanFP = 0.0 var meanFN = 0.0 var meanTN = 0.0 for (i in 0 until numberOfExecutions) { meanTP += confusionMatrix[i][0].toDouble() meanFP += confusionMatrix[i][1].toDouble() meanFN += confusionMatrix[i][2].toDouble() meanTN += confusionMatrix[i][3].toDouble() } meanTP /= numberOfExecutions meanFP /= numberOfExecutions meanFN /= numberOfExecutions meanTN /= numberOfExecutions meanConfusionMatrix.add("$meanTP;$meanFP;$meanFN;$meanTN") } IO.writeToFile("$outputBasePath/xmlb/$bugType.csv", meanConfusionMatrix.toTypedArray()) } /** * Calculate average global confusion matrix. */ val meanConfusionMatrix = mutableListOf("TP;FP;FN;TN") for (confusionMatrix in confusionMatrices) { var meanTP = 0.0 var meanFP = 0.0 var meanFN = 0.0 var meanTN = 0.0 for (i in 0 until numberOfExecutions) { meanTP += confusionMatrix[i][0].toDouble() meanFP += confusionMatrix[i][1].toDouble() meanFN += confusionMatrix[i][2].toDouble() meanTN += confusionMatrix[i][3].toDouble() } meanTP /= numberOfExecutions meanFP /= numberOfExecutions meanFN /= numberOfExecutions meanTN /= numberOfExecutions meanConfusionMatrix.add("$meanTP;$meanFP;$meanFN;$meanTN") } IO.writeToFile("$outputBasePath/simuhaffdata.csv", meanConfusionMatrix.toTypedArray()) }