2 * This file is subject to the terms and conditions of the GNU General Public
3 * License. See the file "COPYING" in the main directory of this archive
6 * Inline assembly cache operations.
8 * Copyright (C) 1996 David S. Miller (dm@engr.sgi.com)
9 * Copyright (C) 1997 - 2002 Ralf Baechle (ralf@gnu.org)
10 * Copyright (C) 2004 Ralf Baechle (ralf@linux-mips.org)
12 #ifndef _ASM_R4KCACHE_H
13 #define _ASM_R4KCACHE_H
16 #include <asm/cacheops.h>
19 * This macro return a properly sign-extended address suitable as base address
20 * for indexed cache operations. Two issues here:
22 * - The MIPS32 and MIPS64 specs permit an implementation to directly derive
23 * the index bits from the virtual address. This breaks with tradition
24 * set by the R4000. To keep unpleassant surprises from happening we pick
25 * an address in KSEG0 / CKSEG0.
26 * - We need a properly sign extended address for 64-bit code. To get away
27 * without ifdefs we let the compiler do it by a type cast.
29 #define INDEX_BASE CKSEG0
31 #define cache_op(op,addr) \
32 __asm__ __volatile__( \
33 " .set noreorder \n" \
34 " .set mips3\n\t \n" \
39 : "i" (op), "m" (*(unsigned char *)(addr)))
41 static inline void flush_icache_line_indexed(unsigned long addr)
43 cache_op(Index_Invalidate_I, addr);
46 static inline void flush_dcache_line_indexed(unsigned long addr)
48 cache_op(Index_Writeback_Inv_D, addr);
51 static inline void flush_scache_line_indexed(unsigned long addr)
53 cache_op(Index_Writeback_Inv_SD, addr);
56 static inline void flush_icache_line(unsigned long addr)
58 cache_op(Hit_Invalidate_I, addr);
61 static inline void flush_dcache_line(unsigned long addr)
63 cache_op(Hit_Writeback_Inv_D, addr);
66 static inline void invalidate_dcache_line(unsigned long addr)
68 cache_op(Hit_Invalidate_D, addr);
71 static inline void invalidate_scache_line(unsigned long addr)
73 cache_op(Hit_Invalidate_SD, addr);
76 static inline void flush_scache_line(unsigned long addr)
78 cache_op(Hit_Writeback_Inv_SD, addr);
82 * The next two are for badland addresses like signal trampolines.
84 static inline void protected_flush_icache_line(unsigned long addr)
92 ".section\t__ex_table,\"a\"\n\t"
96 : "i" (Hit_Invalidate_I), "r" (addr));
100 * R10000 / R12000 hazard - these processors don't support the Hit_Writeback_D
101 * cacheop so we use Hit_Writeback_Inv_D which is supported by all R4000-style
102 * caches. We're talking about one cacheline unnecessarily getting invalidated
103 * here so the penaltiy isn't overly hard.
105 static inline void protected_writeback_dcache_line(unsigned long addr)
107 __asm__ __volatile__(
110 "1:\tcache %0,(%1)\n"
113 ".section\t__ex_table,\"a\"\n\t"
114 STR(PTR)"\t1b,2b\n\t"
117 : "i" (Hit_Writeback_Inv_D), "r" (addr));
120 static inline void protected_writeback_scache_line(unsigned long addr)
122 __asm__ __volatile__(
125 "1:\tcache %0,(%1)\n"
128 ".section\t__ex_table,\"a\"\n\t"
129 STR(PTR)"\t1b,2b\n\t"
132 : "i" (Hit_Writeback_Inv_SD), "r" (addr));
136 * This one is RM7000-specific
138 static inline void invalidate_tcache_page(unsigned long addr)
140 cache_op(Page_Invalidate_T, addr);
143 #define cache16_unroll32(base,op) \
144 __asm__ __volatile__( \
145 " .set noreorder \n" \
147 " cache %1, 0x000(%0); cache %1, 0x010(%0) \n" \
148 " cache %1, 0x020(%0); cache %1, 0x030(%0) \n" \
149 " cache %1, 0x040(%0); cache %1, 0x050(%0) \n" \
150 " cache %1, 0x060(%0); cache %1, 0x070(%0) \n" \
151 " cache %1, 0x080(%0); cache %1, 0x090(%0) \n" \
152 " cache %1, 0x0a0(%0); cache %1, 0x0b0(%0) \n" \
153 " cache %1, 0x0c0(%0); cache %1, 0x0d0(%0) \n" \
154 " cache %1, 0x0e0(%0); cache %1, 0x0f0(%0) \n" \
155 " cache %1, 0x100(%0); cache %1, 0x110(%0) \n" \
156 " cache %1, 0x120(%0); cache %1, 0x130(%0) \n" \
157 " cache %1, 0x140(%0); cache %1, 0x150(%0) \n" \
158 " cache %1, 0x160(%0); cache %1, 0x170(%0) \n" \
159 " cache %1, 0x180(%0); cache %1, 0x190(%0) \n" \
160 " cache %1, 0x1a0(%0); cache %1, 0x1b0(%0) \n" \
161 " cache %1, 0x1c0(%0); cache %1, 0x1d0(%0) \n" \
162 " cache %1, 0x1e0(%0); cache %1, 0x1f0(%0) \n" \
169 static inline void blast_dcache16(void)
171 unsigned long start = INDEX_BASE;
172 unsigned long end = start + current_cpu_data.dcache.waysize;
173 unsigned long ws_inc = 1UL << current_cpu_data.dcache.waybit;
174 unsigned long ws_end = current_cpu_data.dcache.ways <<
175 current_cpu_data.dcache.waybit;
176 unsigned long ws, addr;
178 for (ws = 0; ws < ws_end; ws += ws_inc)
179 for (addr = start; addr < end; addr += 0x200)
180 cache16_unroll32(addr|ws,Index_Writeback_Inv_D);
183 static inline void blast_dcache16_page(unsigned long page)
185 unsigned long start = page;
186 unsigned long end = start + PAGE_SIZE;
189 cache16_unroll32(start,Hit_Writeback_Inv_D);
191 } while (start < end);
194 static inline void blast_dcache16_page_indexed(unsigned long page)
196 unsigned long start = page;
197 unsigned long end = start + PAGE_SIZE;
198 unsigned long ws_inc = 1UL << current_cpu_data.dcache.waybit;
199 unsigned long ws_end = current_cpu_data.dcache.ways <<
200 current_cpu_data.dcache.waybit;
201 unsigned long ws, addr;
203 for (ws = 0; ws < ws_end; ws += ws_inc)
204 for (addr = start; addr < end; addr += 0x200)
205 cache16_unroll32(addr|ws,Index_Writeback_Inv_D);
208 static inline void blast_icache16(void)
210 unsigned long start = INDEX_BASE;
211 unsigned long end = start + current_cpu_data.icache.waysize;
212 unsigned long ws_inc = 1UL << current_cpu_data.icache.waybit;
213 unsigned long ws_end = current_cpu_data.icache.ways <<
214 current_cpu_data.icache.waybit;
215 unsigned long ws, addr;
217 for (ws = 0; ws < ws_end; ws += ws_inc)
218 for (addr = start; addr < end; addr += 0x200)
219 cache16_unroll32(addr|ws,Index_Invalidate_I);
222 static inline void blast_icache16_page(unsigned long page)
224 unsigned long start = page;
225 unsigned long end = start + PAGE_SIZE;
228 cache16_unroll32(start,Hit_Invalidate_I);
230 } while (start < end);
233 static inline void blast_icache16_page_indexed(unsigned long page)
235 unsigned long start = page;
236 unsigned long end = start + PAGE_SIZE;
237 unsigned long ws_inc = 1UL << current_cpu_data.icache.waybit;
238 unsigned long ws_end = current_cpu_data.icache.ways <<
239 current_cpu_data.icache.waybit;
240 unsigned long ws, addr;
242 for (ws = 0; ws < ws_end; ws += ws_inc)
243 for (addr = start; addr < end; addr += 0x200)
244 cache16_unroll32(addr|ws,Index_Invalidate_I);
247 static inline void blast_scache16(void)
249 unsigned long start = INDEX_BASE;
250 unsigned long end = start + current_cpu_data.scache.waysize;
251 unsigned long ws_inc = 1UL << current_cpu_data.scache.waybit;
252 unsigned long ws_end = current_cpu_data.scache.ways <<
253 current_cpu_data.scache.waybit;
254 unsigned long ws, addr;
256 for (ws = 0; ws < ws_end; ws += ws_inc)
257 for (addr = start; addr < end; addr += 0x200)
258 cache16_unroll32(addr|ws,Index_Writeback_Inv_SD);
261 static inline void blast_scache16_page(unsigned long page)
263 unsigned long start = page;
264 unsigned long end = page + PAGE_SIZE;
267 cache16_unroll32(start,Hit_Writeback_Inv_SD);
269 } while (start < end);
272 static inline void blast_scache16_page_indexed(unsigned long page)
274 unsigned long start = page;
275 unsigned long end = start + PAGE_SIZE;
276 unsigned long ws_inc = 1UL << current_cpu_data.scache.waybit;
277 unsigned long ws_end = current_cpu_data.scache.ways <<
278 current_cpu_data.scache.waybit;
279 unsigned long ws, addr;
281 for (ws = 0; ws < ws_end; ws += ws_inc)
282 for (addr = start; addr < end; addr += 0x200)
283 cache16_unroll32(addr|ws,Index_Writeback_Inv_SD);
286 #define cache32_unroll32(base,op) \
287 __asm__ __volatile__( \
288 " .set noreorder \n" \
290 " cache %1, 0x000(%0); cache %1, 0x020(%0) \n" \
291 " cache %1, 0x040(%0); cache %1, 0x060(%0) \n" \
292 " cache %1, 0x080(%0); cache %1, 0x0a0(%0) \n" \
293 " cache %1, 0x0c0(%0); cache %1, 0x0e0(%0) \n" \
294 " cache %1, 0x100(%0); cache %1, 0x120(%0) \n" \
295 " cache %1, 0x140(%0); cache %1, 0x160(%0) \n" \
296 " cache %1, 0x180(%0); cache %1, 0x1a0(%0) \n" \
297 " cache %1, 0x1c0(%0); cache %1, 0x1e0(%0) \n" \
298 " cache %1, 0x200(%0); cache %1, 0x220(%0) \n" \
299 " cache %1, 0x240(%0); cache %1, 0x260(%0) \n" \
300 " cache %1, 0x280(%0); cache %1, 0x2a0(%0) \n" \
301 " cache %1, 0x2c0(%0); cache %1, 0x2e0(%0) \n" \
302 " cache %1, 0x300(%0); cache %1, 0x320(%0) \n" \
303 " cache %1, 0x340(%0); cache %1, 0x360(%0) \n" \
304 " cache %1, 0x380(%0); cache %1, 0x3a0(%0) \n" \
305 " cache %1, 0x3c0(%0); cache %1, 0x3e0(%0) \n" \
312 static inline void blast_dcache32(void)
314 unsigned long start = INDEX_BASE;
315 unsigned long end = start + current_cpu_data.dcache.waysize;
316 unsigned long ws_inc = 1UL << current_cpu_data.dcache.waybit;
317 unsigned long ws_end = current_cpu_data.dcache.ways <<
318 current_cpu_data.dcache.waybit;
319 unsigned long ws, addr;
321 for (ws = 0; ws < ws_end; ws += ws_inc)
322 for (addr = start; addr < end; addr += 0x400)
323 cache32_unroll32(addr|ws,Index_Writeback_Inv_D);
326 static inline void blast_dcache32_page(unsigned long page)
328 unsigned long start = page;
329 unsigned long end = start + PAGE_SIZE;
332 cache32_unroll32(start,Hit_Writeback_Inv_D);
334 } while (start < end);
337 static inline void blast_dcache32_page_indexed(unsigned long page)
339 unsigned long start = page;
340 unsigned long end = start + PAGE_SIZE;
341 unsigned long ws_inc = 1UL << current_cpu_data.dcache.waybit;
342 unsigned long ws_end = current_cpu_data.dcache.ways <<
343 current_cpu_data.dcache.waybit;
344 unsigned long ws, addr;
346 for (ws = 0; ws < ws_end; ws += ws_inc)
347 for (addr = start; addr < end; addr += 0x400)
348 cache32_unroll32(addr|ws,Index_Writeback_Inv_D);
351 static inline void blast_icache32(void)
353 unsigned long start = INDEX_BASE;
354 unsigned long end = start + current_cpu_data.icache.waysize;
355 unsigned long ws_inc = 1UL << current_cpu_data.icache.waybit;
356 unsigned long ws_end = current_cpu_data.icache.ways <<
357 current_cpu_data.icache.waybit;
358 unsigned long ws, addr;
360 for (ws = 0; ws < ws_end; ws += ws_inc)
361 for (addr = start; addr < end; addr += 0x400)
362 cache32_unroll32(addr|ws,Index_Invalidate_I);
365 static inline void blast_icache32_page(unsigned long page)
367 unsigned long start = page;
368 unsigned long end = start + PAGE_SIZE;
371 cache32_unroll32(start,Hit_Invalidate_I);
373 } while (start < end);
376 static inline void blast_icache32_page_indexed(unsigned long page)
378 unsigned long start = page;
379 unsigned long end = start + PAGE_SIZE;
380 unsigned long ws_inc = 1UL << current_cpu_data.icache.waybit;
381 unsigned long ws_end = current_cpu_data.icache.ways <<
382 current_cpu_data.icache.waybit;
383 unsigned long ws, addr;
385 for (ws = 0; ws < ws_end; ws += ws_inc)
386 for (addr = start; addr < end; addr += 0x400)
387 cache32_unroll32(addr|ws,Index_Invalidate_I);
390 static inline void blast_scache32(void)
392 unsigned long start = INDEX_BASE;
393 unsigned long end = start + current_cpu_data.scache.waysize;
394 unsigned long ws_inc = 1UL << current_cpu_data.scache.waybit;
395 unsigned long ws_end = current_cpu_data.scache.ways <<
396 current_cpu_data.scache.waybit;
397 unsigned long ws, addr;
399 for (ws = 0; ws < ws_end; ws += ws_inc)
400 for (addr = start; addr < end; addr += 0x400)
401 cache32_unroll32(addr|ws,Index_Writeback_Inv_SD);
404 static inline void blast_scache32_page(unsigned long page)
406 unsigned long start = page;
407 unsigned long end = page + PAGE_SIZE;
410 cache32_unroll32(start,Hit_Writeback_Inv_SD);
412 } while (start < end);
415 static inline void blast_scache32_page_indexed(unsigned long page)
417 unsigned long start = page;
418 unsigned long end = start + PAGE_SIZE;
419 unsigned long ws_inc = 1UL << current_cpu_data.scache.waybit;
420 unsigned long ws_end = current_cpu_data.scache.ways <<
421 current_cpu_data.scache.waybit;
422 unsigned long ws, addr;
424 for (ws = 0; ws < ws_end; ws += ws_inc)
425 for (addr = start; addr < end; addr += 0x400)
426 cache32_unroll32(addr|ws,Index_Writeback_Inv_SD);
429 #define cache64_unroll32(base,op) \
430 __asm__ __volatile__( \
431 " .set noreorder \n" \
433 " cache %1, 0x000(%0); cache %1, 0x040(%0) \n" \
434 " cache %1, 0x080(%0); cache %1, 0x0c0(%0) \n" \
435 " cache %1, 0x100(%0); cache %1, 0x140(%0) \n" \
436 " cache %1, 0x180(%0); cache %1, 0x1c0(%0) \n" \
437 " cache %1, 0x200(%0); cache %1, 0x240(%0) \n" \
438 " cache %1, 0x280(%0); cache %1, 0x2c0(%0) \n" \
439 " cache %1, 0x300(%0); cache %1, 0x340(%0) \n" \
440 " cache %1, 0x380(%0); cache %1, 0x3c0(%0) \n" \
441 " cache %1, 0x400(%0); cache %1, 0x440(%0) \n" \
442 " cache %1, 0x480(%0); cache %1, 0x4c0(%0) \n" \
443 " cache %1, 0x500(%0); cache %1, 0x540(%0) \n" \
444 " cache %1, 0x580(%0); cache %1, 0x5c0(%0) \n" \
445 " cache %1, 0x600(%0); cache %1, 0x640(%0) \n" \
446 " cache %1, 0x680(%0); cache %1, 0x6c0(%0) \n" \
447 " cache %1, 0x700(%0); cache %1, 0x740(%0) \n" \
448 " cache %1, 0x780(%0); cache %1, 0x7c0(%0) \n" \
455 static inline void blast_icache64(void)
457 unsigned long start = INDEX_BASE;
458 unsigned long end = start + current_cpu_data.icache.waysize;
459 unsigned long ws_inc = 1UL << current_cpu_data.icache.waybit;
460 unsigned long ws_end = current_cpu_data.icache.ways <<
461 current_cpu_data.icache.waybit;
462 unsigned long ws, addr;
464 for (ws = 0; ws < ws_end; ws += ws_inc)
465 for (addr = start; addr < end; addr += 0x800)
466 cache64_unroll32(addr|ws,Index_Invalidate_I);
469 static inline void blast_icache64_page(unsigned long page)
471 unsigned long start = page;
472 unsigned long end = start + PAGE_SIZE;
475 cache64_unroll32(start,Hit_Invalidate_I);
477 } while (start < end);
480 static inline void blast_icache64_page_indexed(unsigned long page)
482 unsigned long start = page;
483 unsigned long end = start + PAGE_SIZE;
484 unsigned long ws_inc = 1UL << current_cpu_data.icache.waybit;
485 unsigned long ws_end = current_cpu_data.icache.ways <<
486 current_cpu_data.icache.waybit;
487 unsigned long ws, addr;
489 for (ws = 0; ws < ws_end; ws += ws_inc)
490 for (addr = start; addr < end; addr += 0x800)
491 cache64_unroll32(addr|ws,Index_Invalidate_I);
494 static inline void blast_scache64(void)
496 unsigned long start = INDEX_BASE;
497 unsigned long end = start + current_cpu_data.scache.waysize;
498 unsigned long ws_inc = 1UL << current_cpu_data.scache.waybit;
499 unsigned long ws_end = current_cpu_data.scache.ways <<
500 current_cpu_data.scache.waybit;
501 unsigned long ws, addr;
503 for (ws = 0; ws < ws_end; ws += ws_inc)
504 for (addr = start; addr < end; addr += 0x800)
505 cache64_unroll32(addr|ws,Index_Writeback_Inv_SD);
508 static inline void blast_scache64_page(unsigned long page)
510 unsigned long start = page;
511 unsigned long end = page + PAGE_SIZE;
514 cache64_unroll32(start,Hit_Writeback_Inv_SD);
516 } while (start < end);
519 static inline void blast_scache64_page_indexed(unsigned long page)
521 unsigned long start = page;
522 unsigned long end = start + PAGE_SIZE;
523 unsigned long ws_inc = 1UL << current_cpu_data.scache.waybit;
524 unsigned long ws_end = current_cpu_data.scache.ways <<
525 current_cpu_data.scache.waybit;
526 unsigned long ws, addr;
528 for (ws = 0; ws < ws_end; ws += ws_inc)
529 for (addr = start; addr < end; addr += 0x800)
530 cache64_unroll32(addr|ws,Index_Writeback_Inv_SD);
533 #define cache128_unroll32(base,op) \
534 __asm__ __volatile__( \
535 " .set noreorder \n" \
537 " cache %1, 0x000(%0); cache %1, 0x080(%0) \n" \
538 " cache %1, 0x100(%0); cache %1, 0x180(%0) \n" \
539 " cache %1, 0x200(%0); cache %1, 0x280(%0) \n" \
540 " cache %1, 0x300(%0); cache %1, 0x380(%0) \n" \
541 " cache %1, 0x400(%0); cache %1, 0x480(%0) \n" \
542 " cache %1, 0x500(%0); cache %1, 0x580(%0) \n" \
543 " cache %1, 0x600(%0); cache %1, 0x680(%0) \n" \
544 " cache %1, 0x700(%0); cache %1, 0x780(%0) \n" \
545 " cache %1, 0x800(%0); cache %1, 0x880(%0) \n" \
546 " cache %1, 0x900(%0); cache %1, 0x980(%0) \n" \
547 " cache %1, 0xa00(%0); cache %1, 0xa80(%0) \n" \
548 " cache %1, 0xb00(%0); cache %1, 0xb80(%0) \n" \
549 " cache %1, 0xc00(%0); cache %1, 0xc80(%0) \n" \
550 " cache %1, 0xd00(%0); cache %1, 0xd80(%0) \n" \
551 " cache %1, 0xe00(%0); cache %1, 0xe80(%0) \n" \
552 " cache %1, 0xf00(%0); cache %1, 0xf80(%0) \n" \
559 static inline void blast_scache128(void)
561 unsigned long start = INDEX_BASE;
562 unsigned long end = start + current_cpu_data.scache.waysize;
563 unsigned long ws_inc = 1UL << current_cpu_data.scache.waybit;
564 unsigned long ws_end = current_cpu_data.scache.ways <<
565 current_cpu_data.scache.waybit;
566 unsigned long ws, addr;
568 for (ws = 0; ws < ws_end; ws += ws_inc)
569 for (addr = start; addr < end; addr += 0x1000)
570 cache128_unroll32(addr|ws,Index_Writeback_Inv_SD);
573 static inline void blast_scache128_page(unsigned long page)
575 unsigned long start = page;
576 unsigned long end = page + PAGE_SIZE;
579 cache128_unroll32(start,Hit_Writeback_Inv_SD);
581 } while (start < end);
584 static inline void blast_scache128_page_indexed(unsigned long page)
586 unsigned long start = page;
587 unsigned long end = start + PAGE_SIZE;
588 unsigned long ws_inc = 1UL << current_cpu_data.scache.waybit;
589 unsigned long ws_end = current_cpu_data.scache.ways <<
590 current_cpu_data.scache.waybit;
591 unsigned long ws, addr;
593 for (ws = 0; ws < ws_end; ws += ws_inc)
594 for (addr = start; addr < end; addr += 0x1000)
595 cache128_unroll32(addr|ws,Index_Writeback_Inv_SD);
598 #endif /* _ASM_R4KCACHE_H */