Recover dcache

src/main/scala/xiangshan/XSCore.scala

@@ -210,8 +210,9 @@ trait HasXSParameter {
   )
 
   val dcacheParameters = DCacheParameters(
-    tagECC = Some("none"),
-    dataECC = Some("none"),
+    tagECC = Some("secded"),
+    dataECC = Some("secded"),
+    replacer = Some("setplru"),
     nMissEntries = 16,
     nProbeEntries = 16,
     nReleaseEntries = 16,

src/main/scala/xiangshan/backend/issue/ReservationStation.scala

@@ -212,7 +212,7 @@ class ReservationStationSelect
   val fastPortsCnt = fastPortsCfg.size
   val slowPortsCnt = slowPortsCfg.size
   require(nonBlocked==fastWakeup)
-  val replayDelay = VecInit(Seq(5, 10, 25, 25).map(_.U(5.W)))
+  val replayDelay = VecInit(Seq(5, 10, 40, 40).map(_.U(6.W)))
 
   val io = IO(new Bundle {
     val redirect = Flipped(ValidIO(new Redirect))

src/main/scala/xiangshan/cache/DCache.scala

@@ -3,7 +3,7 @@ package xiangshan.cache
 import chisel3._
 import chisel3.util._
 import freechips.rocketchip.tilelink.{ClientMetadata, TLClientParameters, TLEdgeOut}
-import utils.{Code, RandomReplacement, XSDebug, SRAMTemplate}
+import utils.{Code, ReplacementPolicy, XSDebug, SRAMTemplate, ParallelOR}
 
 import scala.math.max
 
@@ -15,10 +15,11 @@ case class DCacheParameters
 (
     nSets: Int = 64,
     nWays: Int = 8,
-    rowBits: Int = 64,
+    rowBits: Int = 128,
     nTLBEntries: Int = 32,
     tagECC: Option[String] = None,
     dataECC: Option[String] = None,
+    replacer: Option[String] = Some("random"),
     nMissEntries: Int = 1,
     nProbeEntries: Int = 1,
     nReleaseEntries: Int = 1,
@@ -30,8 +31,7 @@ case class DCacheParameters
 
   def tagCode: Code = Code.fromString(tagECC)
   def dataCode: Code = Code.fromString(dataECC)
-
-  def replacement = new RandomReplacement(nWays)
+  def replacement = ReplacementPolicy.fromString(replacer, nWays, nSets)
 }
 
 trait HasDCacheParameters extends HasL1CacheParameters {
@@ -60,7 +60,7 @@ trait HasDCacheParameters extends HasL1CacheParameters {
   require(full_divide(beatBits, rowBits), s"beatBits($beatBits) must be multiple of rowBits($rowBits)")
   // this is a VIPT L1 cache
   require(pgIdxBits >= untagBits, s"page aliasing problem: pgIdxBits($pgIdxBits) < untagBits($untagBits)")
-  require(rowWords == 1, "Our DCache Implementation assumes rowWords == 1")
+  // require(rowWords == 1, "Our DCache Implementation assumes rowWords == 1")
 }
 
 abstract class DCacheModule extends L1CacheModule
@@ -103,15 +103,20 @@ class L1DataReadReq extends DCacheBundle {
 
 // Now, we can write a cache-block in a single cycle
 class L1DataWriteReq extends L1DataReadReq {
-  val wmask  = Vec(blockRows, Bits(rowWords.W))
-  val data   = Vec(blockRows, Bits(encRowBits.W))
+  val wmask  = Bits(blockRows.W)
+  val data   = Vec(blockRows, Bits(rowBits.W))
+}
+
+class ReplacementAccessBundle extends DCacheBundle {
+  val set = UInt(log2Up(nSets).W)
+  val way = UInt(log2Up(nWays).W)
 }
 
-abstract class AbstractDataArray extends DCacheModule {
+abstract class TransposeAbstractDataArray extends DCacheModule {
   val io = IO(new DCacheBundle {
     val read  = Vec(LoadPipelineWidth, Flipped(DecoupledIO(new L1DataReadReq)))
     val write = Flipped(DecoupledIO(new L1DataWriteReq))
-    val resp  = Output(Vec(LoadPipelineWidth, Vec(nWays, Vec(blockRows, Bits(encRowBits.W)))))
+    val resp = Output(Vec(LoadPipelineWidth, Vec(blockRows, Bits(encRowBits.W))))
     val nacks = Output(Vec(LoadPipelineWidth, Bool()))
   })
 
@@ -141,10 +146,8 @@ abstract class AbstractDataArray extends DCacheModule {
   def dumpResp() = {
     (0 until LoadPipelineWidth) map { w =>
       XSDebug(s"DataArray ReadResp channel: $w\n")
-      (0 until nWays) map { i =>
-        (0 until blockRows) map { r =>
-          XSDebug(s"way: $i cycle: $r data: %x\n", io.resp(w)(i)(r))
-        }
+      (0 until blockRows) map { r =>
+        XSDebug(s"cycle: $r data: %x\n", io.resp(w)(r))
       }
     }
   }
@@ -165,61 +168,131 @@ abstract class AbstractDataArray extends DCacheModule {
   }
 }
 
-class DuplicatedDataArray extends AbstractDataArray
-{
+class TransposeDuplicatedDataArray extends TransposeAbstractDataArray {
   val singlePort = true
-  // write is always ready
-  io.write.ready := true.B
+  val readHighPriority = true
+  def eccBits = encWordBits - wordBits
+
+  def getECCFromEncWord(encWord: UInt) = {
+    require(encWord.getWidth == encWordBits)
+    encWord(encWordBits - 1, wordBits)
+  }
+
+  def getECCFromRow(row: UInt) = {
+    require(row.getWidth == rowBits)
+    VecInit((0 until rowWords).map { w =>
+      val word = row(wordBits * (w + 1) - 1, wordBits * w)
+      getECCFromEncWord(cacheParams.dataCode.encode(word))
+    })
+  }
+
   val waddr = (io.write.bits.addr >> blockOffBits).asUInt()
+  val raddrs = io.read.map(r => (r.bits.addr >> blockOffBits).asUInt)
+  io.write.ready := (if (readHighPriority) {
+    if (singlePort) {
+      !VecInit(io.read.map(_.valid)).asUInt.orR
+    } else {
+      !(Cat(io.read.zipWithIndex.map { case (r, i) => r.valid && raddrs(i) === waddr }).orR)
+    }
+  } else {
+    true.B
+  })
+
   for (j <- 0 until LoadPipelineWidth) {
-    val raddr = (io.read(j).bits.addr >> blockOffBits).asUInt()
+    val raddr = raddrs(j)
+    val rmask = io.read(j).bits.rmask
 
     // for single port SRAM, do not allow read and write in the same cycle
     // for dual port SRAM, raddr === waddr is undefined behavior
     val rwhazard = if(singlePort) io.write.valid else io.write.valid && waddr === raddr
-    io.read(j).ready := !rwhazard
-
-    for (w <- 0 until nWays) {
-      for (r <- 0 until blockRows) {
-        val resp = Seq.fill(rowWords)(Wire(Bits(encWordBits.W)))
-        io.resp(j)(w)(r) := Cat((0 until rowWords).reverse map (k => resp(k)))
-
-        for (k <- 0 until rowWords) {
-          val array = Module(new SRAMTemplate(
-            Bits(encWordBits.W),
-            set=nSets,
-            way=1,
-            shouldReset=false,
-            holdRead=false,
-            singlePort=singlePort
-          ))
-          // data write
-          val wen = io.write.valid && io.write.bits.way_en(w) && io.write.bits.wmask(r)(k)
-          array.io.w.req.valid := wen
-          array.io.w.req.bits.apply(
-            setIdx=waddr,
-            data=io.write.bits.data(r)(encWordBits*(k+1)-1,encWordBits*k),
-            waymask=1.U
-          )
-
-          // data read
-          val ren = io.read(j).valid && io.read(j).bits.way_en(w) && io.read(j).bits.rmask(r)
-          array.io.r.req.valid := ren
-          array.io.r.req.bits.apply(setIdx=raddr)
-          resp(k) := array.io.r.resp.data(0)
+    io.read(j).ready := (if (readHighPriority) true.B else !rwhazard)
+
+    // use way_en to select a way after data read out
+    assert(!(RegNext(io.read(j).fire() && PopCount(io.read(j).bits.way_en) > 1.U)))
+    val way_en = RegNext(io.read(j).bits.way_en)
+
+    for (r <- 0 until blockRows) {
+      val resp = Wire(Vec(rowWords, Vec(nWays, Bits(wordBits.W))))
+      val resp_chosen = Wire(Vec(rowWords, Bits(wordBits.W)))
+      val ecc_resp = Wire(Vec(rowWords, Vec(nWays, Bits(eccBits.W))))
+      val ecc_resp_chosen = Wire(Vec(rowWords, Bits(eccBits.W)))
+
+      val ecc_array = Module(new SRAMTemplate(
+        Vec(rowWords, Bits(eccBits.W)),
+        set = nSets,
+        way = nWays,
+        shouldReset = false,
+        holdRead = false,
+        singlePort = singlePort
+      ))
+
+      ecc_array.io.w.req.valid := io.write.valid && io.write.bits.wmask(r)
+      ecc_array.io.w.req.bits.apply(
+        setIdx = waddr,
+        data = getECCFromRow(io.write.bits.data(r)),
+        waymask = io.write.bits.way_en
+      )
+      when (ecc_array.io.w.req.valid) {
+        XSDebug(p"write in ecc sram ${j.U} row ${r.U}: setIdx=${Hexadecimal(ecc_array.io.w.req.bits.setIdx)} ecc(0)=${Hexadecimal(getECCFromRow(io.write.bits.data(r))(0))} ecc(1)=${Hexadecimal(getECCFromRow(io.write.bits.data(r))(1))} waymask=${Hexadecimal(io.write.bits.way_en)}\n")
+      }
+
+      ecc_array.io.r.req.valid := io.read(j).valid && rmask(r)
+      ecc_array.io.r.req.bits.apply(setIdx = raddr)
+
+      for (w <- 0 until nWays) {
+        val data_array = Module(new SRAMTemplate(
+          Bits(rowBits.W),
+          set = nSets,
+          way = 1,
+          shouldReset = false,
+          holdRead = false,
+          singlePort = singlePort
+        ))
+
+        // data write
+        val wen = io.write.valid && io.write.bits.way_en(w) && io.write.bits.wmask(r)
+        data_array.io.w.req.valid := wen
+        data_array.io.w.req.bits.apply(
+          setIdx = waddr,
+          data = io.write.bits.data(r),
+          waymask = 1.U
+        )
+        when (wen) {
+          XSDebug(p"write in data sram ${j.U} row ${r.U} way ${w.U}: setIdx=${Hexadecimal(data_array.io.w.req.bits.setIdx)} data=${Hexadecimal(io.write.bits.data(r))}\n")
         }
+
+        // data read
+        // read all ways and choose one after resp
+        val ren = io.read(j).valid && rmask(r)
+        data_array.io.r.req.valid := ren
+        data_array.io.r.req.bits.apply(setIdx = raddr)
+        (0 until rowWords).foreach(k => resp(k)(w) := data_array.io.r.resp.data(0)(wordBits * (k + 1) - 1, wordBits * k))
+        (0 until rowWords).foreach(k => ecc_resp(k)(w) := ecc_array.io.r.resp.data(w)(k))
       }
+      for (k <- 0 until rowWords) {
+        resp_chosen(k) := Mux1H(way_en, resp(k))
+        ecc_resp_chosen(k) := Mux1H(way_en, ecc_resp(k))
+        // assert(!RegNext(cacheParams.dataCode.decode(Cat(ecc_resp_chosen(k), resp_chosen(k))).uncorrectable &&
+        //   way_en.orR &&
+        //   RegNext(io.read(j).fire() && rmask(r))))
+      }
+      io.resp(j)(r) := Cat((0 until rowWords).reverse map {k => Cat(ecc_resp_chosen(k), resp_chosen(k))})// resp_chosen.asUInt
+
     }
+
     io.nacks(j) := false.B
   }
 }
 
 class L1MetadataArray(onReset: () => L1Metadata) extends DCacheModule {
   val rstVal = onReset()
+  val metaBits = rstVal.getWidth
+  val encMetaBits = cacheParams.tagCode.width(metaBits)
+
   val io = IO(new Bundle {
     val read = Flipped(Decoupled(new L1MetaReadReq))
     val write = Flipped(Decoupled(new L1MetaWriteReq))
-    val resp = Output(Vec(nWays, new L1Metadata))
+    val resp = Output(Vec(nWays, UInt(encMetaBits.W)))
   })
   val rst_cnt = RegInit(0.U(log2Up(nSets+1).W))
   val rst = rst_cnt < nSets.U
@@ -229,28 +302,25 @@ class L1MetadataArray(onReset: () => L1Metadata) extends DCacheModule {
   val rmask = Mux(rst || (nWays == 1).B, (-1).asSInt, io.read.bits.way_en.asSInt).asBools
   when (rst) { rst_cnt := rst_cnt + 1.U }
 
-  val metaBits = rstVal.getWidth
-  val encMetaBits = cacheParams.tagCode.width(metaBits)
-
   val tag_array = Module(new SRAMTemplate(UInt(encMetaBits.W), set=nSets, way=nWays,
     shouldReset=false, holdRead=false, singlePort=true))
 
   // tag write
-  val wen = rst || io.write.valid
+  val wen = rst || io.write.fire()
   tag_array.io.w.req.valid := wen
   tag_array.io.w.req.bits.apply(
     setIdx=waddr,
     data=cacheParams.tagCode.encode(wdata),
     waymask=VecInit(wmask).asUInt)
 
   // tag read
-  tag_array.io.r.req.valid := io.read.fire()
+  val ren = io.read.fire()
+  tag_array.io.r.req.valid := ren
   tag_array.io.r.req.bits.apply(setIdx=io.read.bits.idx)
-  io.resp := tag_array.io.r.resp.data.map(rdata =>
-      cacheParams.tagCode.decode(rdata).corrected.asTypeOf(rstVal))
+  io.resp := tag_array.io.r.resp.data
 
-  io.read.ready := !wen
-  io.write.ready := !rst
+  io.write.ready := !ren
+  io.read.ready := !rst
 
   def dumpRead() = {
     when (io.read.fire()) {
@@ -266,35 +336,40 @@ class L1MetadataArray(onReset: () => L1Metadata) extends DCacheModule {
     }
   }
 
-  def dumpResp() = {
-    (0 until nWays) map { i =>
-      XSDebug(s"MetaArray Resp: way: $i tag: %x coh: %x\n",
-        io.resp(i).tag, io.resp(i).coh.state)
-    }
-  }
+  // def dumpResp() = {
+  //   (0 until nWays) map { i =>
+  //     XSDebug(s"MetaArray Resp: way: $i tag: %x coh: %x\n",
+  //       io.resp(i).tag, io.resp(i).coh.state)
+  //   }
+  // }
 
   def dump() = {
     dumpRead
     dumpWrite
-    dumpResp
+    // dumpResp
   }
 }
 
 class DuplicatedMetaArray extends DCacheModule {
+  def onReset = L1Metadata(0.U, ClientMetadata.onReset)
+  val metaBits = onReset.getWidth
+  val encMetaBits = cacheParams.tagCode.width(metaBits)
+
   val io = IO(new DCacheBundle {
     val read  = Vec(LoadPipelineWidth, Flipped(DecoupledIO(new L1MetaReadReq)))
     val write = Flipped(DecoupledIO(new L1MetaWriteReq))
-    val resp  = Output(Vec(LoadPipelineWidth, Vec(nWays, new L1Metadata)))
+    val resp  = Output(Vec(LoadPipelineWidth, Vec(nWays, UInt(encMetaBits.W))))
   })
-
-  def onReset = L1Metadata(0.U, ClientMetadata.onReset)
   val meta = Seq.fill(LoadPipelineWidth) { Module(new L1MetadataArray(onReset _)) }
 
   for (w <- 0 until LoadPipelineWidth) {
-    meta(w).io.write <> io.write
+    // meta(w).io.write <> io.write
+    meta(w).io.write.valid := io.write.valid
+    meta(w).io.write.bits := io.write.bits
     meta(w).io.read  <> io.read(w)
     io.resp(w) <> meta(w).io.resp
   }
+  io.write.ready := VecInit(meta.map(_.io.write.ready)).asUInt.andR
 
   def dumpRead() = {
     (0 until LoadPipelineWidth) map { w =>
@@ -312,18 +387,18 @@ class DuplicatedMetaArray extends DCacheModule {
     }
   }
 
-  def dumpResp() = {
-    (0 until LoadPipelineWidth) map { w =>
-      (0 until nWays) map { i =>
-        XSDebug(s"MetaArray Resp: channel: $w way: $i tag: %x coh: %x\n",
-          io.resp(w)(i).tag, io.resp(w)(i).coh.state)
-      }
-    }
-  }
+  // def dumpResp() = {
+  //   (0 until LoadPipelineWidth) map { w =>
+  //     (0 until nWays) map { i =>
+  //       XSDebug(s"MetaArray Resp: channel: $w way: $i tag: %x coh: %x\n",
+  //         io.resp(w)(i).tag, io.resp(w)(i).coh.state)
+  //     }
+  //   }
+  // }
 
   def dump() = {
     dumpRead
     dumpWrite
-    dumpResp
+    // dumpResp
   }
 }