Don't hide sprites behind a 2BPP textbox if they are partially outside of it, and make 2BPP textboxes have priority over objects (#7); Generalize 1bpp/2bpp text state tracking (#10) [reworks 052d246e]

constants/wram_constants.asm

@@ -61,6 +61,7 @@ DEF NUM_FRAMES EQU const_value
 	const_def
 	const FAST_TEXT_DELAY_F ; 0
 	const TEXT_DELAY_F      ; 1
+	const TEXT_2BPP_F       ; 2
 
 ; wGBPrinterBrightness::
 DEF GBPRINTER_LIGHTEST EQU $00

docs/develop/index.md

@@ -15,8 +15,8 @@
 
 #### Textboxes
 
-- **Textbox1bpp**: TextboxBorder + TextboxPalette
+- **Textbox1bpp**: TextboxBorder + TextboxAttributes1bpp
-- **Textbox2bpp**: _OverworldTextbox + TextboxPalette
+- **Textbox2bpp**: _OverworldTextbox + TextboxAttributes2bpp
 - **SpeechTextbox1bpp**: Textbox1bpp with speech location and dimensions
 - **SpeechTextbox2bpp**: Textbox2bpp with speech location and dimensions
 - **ClearTextbox**: Clear the inside of a speech textbox (fill with " ")
@@ -24,7 +24,7 @@
 - **PrintText1bpp**, **FarPrintText1bpp**: SpeechTextbox1bpp + UpdateSprites + ApplyTilemap + ClearTextbox + PrintTextboxText
 - **PrintText2bpp**: SpeechTextbox2bpp + UpdateSprites + ApplyTilemap + ClearTextbox + PrintTextboxText
 - **MapTextbox**: ClearTextbox + redraw tile behind cursor + SafeUpdateSprites + disable OAM update + ApplyTilemap + PrintTextboxText + enable OAM update
-- **MenuBox**: Calls Textbox1bpp or Textbox2bpp, depending on the value at wMenuBoxUse2bppFrame, with menu location and dimensions. wMenuBoxUse2bppFrame, as part of menu data, is cleared (FALSE means 1bpp) by ClearMenuAndWindowData
+- **MenuBox**: Calls Textbox1bpp or Textbox2bpp, depending on the value at wTextboxFlags[TEXT_2BPP_F], with menu location and dimensions.
 
 #### Overworld map scrolling
 

engine/menus/start_menu.asm

@@ -13,6 +13,9 @@
 StartMenu::
 	call ClearMenuAndWindowData
 
+	ld hl, wTextboxFlags
+	res TEXT_2BPP_F, [hl]
+
 	ld de, SFX_MENU
 	call PlaySFX
 
@@ -378,7 +381,7 @@ endr
 	hlcoord 0, 13
 	ld b, 3
 	ld c, 8
-	jp TextboxPalette
+	jp TextboxAttributes1bpp
 
 .IsMenuAccountOn:
 	ld a, [wOptions2]

engine/overworld/events.asm

@@ -110,6 +110,8 @@ EnterMap:
 	ld [wXYComparePointer], a
 	ld [wXYComparePointer + 1], a
 	call SetUpFiveStepWildEncounterCooldown
+	ld hl, wTextboxFlags
+	set TEXT_2BPP_F, [hl]
 	farcall RunMapSetupScript
 	call DisableEvents
 

engine/overworld/map_objects.asm

@@ -2252,14 +2252,14 @@ UpdateObjectFrozen:
 	call CheckObjectOnScreen
 	jr c, SetFacing_Standing
 	call UpdateObjectTile
-	farcall HandleFrozenObjectAction ; no need to farcall
+	call HandleFrozenObjectAction
 	xor a
 	ret
 
 UpdateRespawnedObjectFrozen:
 	call CheckObjectOnScreen
 	jr c, SetFacing_Standing
-	farcall HandleFrozenObjectAction ; no need to farcall
+	call HandleFrozenObjectAction
 	xor a
 	ret
 
@@ -2283,7 +2283,7 @@ UpdateObjectTile:
 	ld hl, OBJECT_TILE
 	add hl, bc
 	ld [hl], a
-	farcall UpdateTallGrassFlags ; no need to farcall
+	call UpdateTallGrassFlags
 	ret
 
 CheckObjectOnScreen:
@@ -2332,7 +2332,7 @@ CheckObjectCoveredByTextbox:
 	cp $f0
 	jr nc, .ok1
 	cp SCREEN_WIDTH_PX
-	jp nc, .nope
+	jp nc, .disappear
 .ok1
 ; Account for objects currently moving left/right.
 	and %00000111
@@ -2365,7 +2365,7 @@ CheckObjectCoveredByTextbox:
 	cp $f0
 	jr nc, .ok4
 	cp SCREEN_HEIGHT_PX
-	jr nc, .nope
+	jr nc, .disappear
 .ok4
 ; Account for objects currently moving up/down.
 	and %00000111
@@ -2419,23 +2419,50 @@ CheckObjectCoveredByTextbox:
 	push bc
 	call Coord2Tile
 	pop bc
-; NPCs disappear if standing on tile $60-$7f (or $e0-$ff),
+; NPCs disappear if standing on tiles FIRST_REGULAR_TEXT_CHAR or above,
 ; since those IDs are for text characters and textbox frames.
+; - if 1bpp text, a single textbox tile overlapping with the sprite makes it disappear.
+; - if 2bpp text, sprite only disappears if ALL overlapping tiles are textbox tiles.
 	ld a, [hl]
 	cp FIRST_REGULAR_TEXT_CHAR
-	jr nc, .nope
+	jr c, .object_not_in_textbox
+
+;.object_in_textbox
+	ld a, [wTextboxFlags]
+	bit TEXT_2BPP_F, a
+	jr z, .disappear
+	jr .ok8
+
+.object_not_in_textbox
+	ld a, [wTextboxFlags]
+	bit TEXT_2BPP_F, a
+	jr nz, .not_disappear
+
 .ok8
 	dec d
 	jr nz, .next
 .ok9
+; while sprites are centered to tiles in the X axis exactly on top of two adjacent horizonal tiles,
+; in the Y axis they occupy 4-8-4 pixels of adjacent vertical tiles instead.
+; this is why we loop here one more time (thrice for regular-sized sprites) than horizontally.
 	dec e
+	ld a, e
+	cp $ff
 	jr nz, .loop
 
-	and a
+; if we managed make it here without returning early, there are only two options:
+; - if 1bpp text, the sprite is wholly outside of a textbox
+; - if 2bpp text, the sprite is wholly inside a textbox
+	ld a, [wTextboxFlags]
+	bit TEXT_2BPP_F, a
+	jr z, .not_disappear
+
+.disappear
+	scf
 	ret
 
-.nope
+.not_disappear
-	scf
+	and a
 	ret
 
 HandleNPCStep::
@@ -2584,7 +2611,7 @@ ResetFollower:
 	cp -1
 	ret z
 	call GetObjectStruct
-	farcall ResetObject ; no need to farcall
+	call ResetObject
 	ld a, -1
 	ld [wObjectFollow_Follower], a
 	ret

engine/overworld/scripting.asm

@@ -383,8 +383,6 @@ Script_promptbutton:
 	ret
 
 Script_yesorno:
-	ld a, TRUE
-	ld [wMenuBoxUse2bppFrame], a
 	call YesNoBox
 	ld a, FALSE
 	jr c, .no
@@ -425,8 +423,6 @@ Script_closepokepic:
 	ret
 
 Script_verticalmenu:
-	ld a, TRUE
-	ld [wMenuBoxUse2bppFrame], a
 	ld a, [wScriptBank]
 	ld hl, VerticalMenu
 	rst FarCall
@@ -438,8 +434,6 @@ Script_verticalmenu:
 	ret
 
 Script__2dmenu:
-	ld a, TRUE
-	ld [wMenuBoxUse2bppFrame], a
 	ld a, [wScriptBank]
 	ld hl, _2DMenu
 	rst FarCall

home/menu.asm

@@ -209,8 +209,8 @@ MenuBox::
 	call GetMenuBoxDims
 	dec b
 	dec c
-	ld a, [wMenuBoxUse2bppFrame]
+	ld a, [wTextboxFlags]
-	and a
+	bit TEXT_2BPP_F, a
 	jp z, Textbox1bpp
 	ld d, h
 	ld e, l

home/text.asm

@@ -72,7 +72,7 @@ Textbox1bpp::
 	call TextboxBorder
 	pop hl
 	pop bc
-	jr TextboxPalette
+	jr TextboxAttributes1bpp
 
 TextboxBorder::
 	; Top
@@ -120,15 +120,23 @@ TextboxBorder::
 	jr nz, .loop
 	ret
 
-TextboxPalette::
+TextboxAttributes1bpp::
-; Fill text box width c height b at hl with pal 7
+; Fill text box width c height b at hl with PAL_BG_TEXT
+	ld a, PAL_BG_TEXT
+	jr _TextboxAttributes
+
+TextboxAttributes2bpp::
+; Fill text box width c height b at hl with PAL_BG_TEXT | PRIORITY
+	ld a, PAL_BG_TEXT | PRIORITY
+	; fallthrough
+
+_TextboxAttributes:
 	ld de, wAttrmap - wTilemap
 	add hl, de
 	inc b
 	inc b
 	inc c
 	inc c
-	ld a, PAL_BG_TEXT
 .col
 	push bc
 	push hl
@@ -155,7 +163,7 @@ Textbox2bpp::
 	farcall _OverworldTextbox
 	pop hl
 	pop bc
-	jp TextboxPalette
+	jp TextboxAttributes2bpp
 
 RadioTerminator::
 	ld hl, .stop

home/window.asm

@@ -46,6 +46,9 @@ OpenText2bpp::
 	ld a, BANK(ReanchorBGMap_NoOAMUpdate)
 	rst Bankswitch
 
+	ld hl, wTextboxFlags
+	set TEXT_2BPP_F, [hl]
+
 	; assumes that the overworld 2bpp font and frame are loaded when calling this
 	call ReanchorBGMap_NoOAMUpdate ; anchor bgmap
 	call SpeechTextbox2bpp
@@ -64,6 +67,9 @@ OpenText1bpp::
 	ld a, BANK(ReanchorBGMap_NoOAMUpdate) ; aka BANK(LoadFont_NoOAMUpdate)
 	rst Bankswitch
 
+	ld hl, wTextboxFlags
+	res TEXT_2BPP_F, [hl]
+
 	call ReanchorBGMap_NoOAMUpdate ; anchor bgmap
 	call SpeechTextbox1bpp
 	call HDMATransferTilemapAndAttrmap_OpenAndCloseMenu ; transfer bgmap

ram/wram.asm

@@ -1192,8 +1192,7 @@ wMenuCursorY:: db
 wMenuCursorX:: db
 wCursorOffCharacter:: db
 wCursorCurrentTile:: dw
-wMenuBoxUse2bppFrame:: db
+	ds 3
-	ds 2
 wMoreMenuDataEnd::
 
 wOverworldDelay:: db
@@ -1253,7 +1252,8 @@ wTextboxFrame::
 	db
 wTextboxFlags::
 ; bit 0: 1-frame text delay
-; bit 4: no text delay
+; bit 1: no text delay
+; bit 2: 2bpp textbox (and text)
 	db
 wGBPrinterBrightness::
 ; bit 0-6: brightness