Sync with https://github.com/andrew6180/TradeSkillMaster

TradeSkillMaster_AuctionDB/Locale/deDE.lua

@@ -14,7 +14,7 @@ local L = LibStub("AceLocale-3.0"):NewLocale("TradeSkillMaster_AuctionDB", "deDE
 if not L then return end
 
 L["A full auction house scan will scan every item on the auction house but is far slower than a GetAll scan. Expect this scan to take several minutes or longer."] = "Ein voller Auktionshausscan wird jedes einzelne Item im Auktionshaus scannen, ist aber sehr viel langsamer als der GetAll-Scan. Erwarte, dass es mehrere Minuten dauert oder länger."
-L["A GetAll scan is the fastest in-game method for scanning every item on the auction house. However, there are many possible bugs on Blizzard's end with it including the chance for it to disconnect you from the game. Also, it has a 15 minute cooldown."] = "Ein GetAll-Scan ist die schnellste Methode, um in-game alle Gegenstände im Auktionshaus zu scannen. Allerdings gibt es viele mögliche Bugs seitens Blizzard die auftreten können, inklusive der Möglichkeit, dass deine Verbindung zum Spiel getrennt wird. Außerdem gibt es einen 15-Minuten-Cooldown."
+L["A GetAll scan is the fastest in-game method for scanning every item on the auction house. However, there are many possible bugs on Blizzard's end with it including the chance for it to disconnect you from the game. Also, it has a 15 minute cooldown. You can disable the GetAll button via TSM's AuctionDB options if this feature doesn't work well on your server."] = "Ein GetAll-Scan ist die schnellste Methode, um in-game alle Gegenstände im Auktionshaus zu scannen. Allerdings gibt es viele mögliche Bugs seitens Blizzard die auftreten können, inklusive der Möglichkeit, dass deine Verbindung zum Spiel getrennt wird. Außerdem gibt es einen 15-Minuten-Cooldown. You can disable the GetAll button via TSM's AuctionDB options if this feature doesn't work well on your server." -- Needs review
 L["Any items in the AuctionDB database that contain the search phrase in their names will be displayed."] = "Es werden alle Gegenstände in der \"AuctionDB\" Datenbank angezeigt, deren Namen mit der Sucheingabe übereinstimmen."
 L["Are you sure you want to clear your AuctionDB data?"] = "Sind Sie sicher, dass Sie die \"AuctionDB\" Daten löschen wollen?"
 L["Ascending"] = "Aufsteigend"

TradeSkillMaster_AuctionDB/Locale/enUS.lua

@@ -14,7 +14,7 @@ local L = LibStub("AceLocale-3.0"):NewLocale("TradeSkillMaster_AuctionDB", "enUS
 if not L then return end
 
 L["%s ago"] = true
-L["A GetAll scan is the fastest in-game method for scanning every item on the auction house. However, there are many possible bugs on Blizzard's end with it including the chance for it to disconnect you from the game. Also, it has a 15 minute cooldown."] = true
+L["A GetAll scan is the fastest in-game method for scanning every item on the auction house. However, there are many possible bugs on Blizzard's end with it including the chance for it to disconnect you from the game. Also, it has a 15 minute cooldown. You can disable the GetAll button via TSM's AuctionDB options if this feature doesn't work well on your server."] = true
 L["A full auction house scan will scan every item on the auction house but is far slower than a GetAll scan. Expect this scan to take several minutes or longer."] = true
 L["Any items in the AuctionDB database that contain the search phrase in their names will be displayed."] = true
 L["Are you sure you want to clear your AuctionDB data?"] = true
@@ -23,6 +23,7 @@ L["AuctionDB - Market Value"] = true
 L["AuctionDB - Minimum Buyout"] = true
 L["Can't run a GetAll scan right now."] = true
 L["Descending"] = true
+L["Disable \"GetAll\" Auction Scans"] = true
 L["Display lowest buyout value seen in the last scan in tooltip."] = true
 L["Display market value in tooltip."] = true
 L["Done Scanning"] = true
@@ -31,6 +32,7 @@ L["Enable display of AuctionDB data in tooltip."] = true
 L["GetAll scan did not run successfully due to issues on Blizzard's end. Using the TSM application for your scans is recommended."] = true
 L["Hide poor quality items"] = true
 L["If checked, AuctionDB will add a tab to the AH to allow for in-game scans. If you are using the TSM app exclusively for your scans, you may want to hide it by unchecking this option. This option requires a reload to take effect."] = true
+L["If checked, AuctionDB will not perform \"GetAll\" scans. This is useful if your server doesn't return all auctions in its \"GetAll\" results, which means that you'll get incorrect market value calculations for all items. If you're playing on such servers, it's best to disable the \"GetAll\" feature to avoid accidentally polluting your price database with incorrect data. This option takes effect immediately, but requires a reload to completely hide the \"Run GetAll Scan\" button."] = true
 L["If checked, poor quality items won't be shown in the search results."] = true
 L["If checked, the lowest buyout value seen in the last scan of the item will be displayed."] = true
 L["If checked, the market value of the item will be displayed"] = true
@@ -72,7 +74,9 @@ L["Result Order:"] = true
 L["Run Full Scan"] = true
 L["Run GetAll Scan"] = true
 L["Running query..."] = true
+L["Running query... Server not responding due to throttling? Try again later..."] = true
 L["Scan Selected Groups"] = true
+L["Scanning %d / %d (Page 1 / ?)"] = true
 L["Scanning %d / %d (Page %d / %d)"] = true
 L["Scanning page %s/%s"] = true
 L["Scanning the auction house in game is no longer necessary!"] = true
@@ -88,4 +92,5 @@ L["This will do a slow auction house scan of every item in the selected groups a
 L["Use the search box and category filters above to search the AuctionDB data."] = true
 L["You can filter the results by item subtype by using this dropdown. For example, if you want to search for all herbs, you would select \"Trade Goods\" in the item type dropdown and \"Herbs\" in this dropdown."] = true
 L["You can filter the results by item type by using this dropdown. For example, if you want to search for all herbs, you would select \"Trade Goods\" in this dropdown and \"Herbs\" as the subtype filter."] = true
-L["You can use this page to lookup an item or group of items in the AuctionDB database. Note that this does not perform a live search of the AH."] = true
+L["You can use this page to lookup an item or group of items in the AuctionDB database. Note that this does not perform a live search of the AH."] = true
+L["You have disabled GetAll scans via AuctionDB's options."] = true

TradeSkillMaster_AuctionDB/Locale/esES.lua

@@ -14,7 +14,7 @@ local L = LibStub("AceLocale-3.0"):NewLocale("TradeSkillMaster_AuctionDB", "esES
 if not L then return end
 
 -- L["A full auction house scan will scan every item on the auction house but is far slower than a GetAll scan. Expect this scan to take several minutes or longer."] = ""
--- L["A GetAll scan is the fastest in-game method for scanning every item on the auction house. However, there are many possible bugs on Blizzard's end with it including the chance for it to disconnect you from the game. Also, it has a 15 minute cooldown."] = ""
+-- L["A GetAll scan is the fastest in-game method for scanning every item on the auction house. However, there are many possible bugs on Blizzard's end with it including the chance for it to disconnect you from the game. Also, it has a 15 minute cooldown. You can disable the GetAll button via TSM's AuctionDB options if this feature doesn't work well on your server."] = ""
 L["Any items in the AuctionDB database that contain the search phrase in their names will be displayed."] = "Todos los artículos en la base de datos de AuctionDB que contienen la frase de búsqueda en su nombre en la pantalla."
 L["Are you sure you want to clear your AuctionDB data?"] = "¿Está seguro que desea borrar los datos AuctionDB?"
 L["Ascending"] = "Ascendente"

TradeSkillMaster_AuctionDB/Locale/esMX.lua

@@ -14,7 +14,7 @@ local L = LibStub("AceLocale-3.0"):NewLocale("TradeSkillMaster_AuctionDB", "esMX
 if not L then return end
 
 -- L["A full auction house scan will scan every item on the auction house but is far slower than a GetAll scan. Expect this scan to take several minutes or longer."] = ""
--- L["A GetAll scan is the fastest in-game method for scanning every item on the auction house. However, there are many possible bugs on Blizzard's end with it including the chance for it to disconnect you from the game. Also, it has a 15 minute cooldown."] = ""
+-- L["A GetAll scan is the fastest in-game method for scanning every item on the auction house. However, there are many possible bugs on Blizzard's end with it including the chance for it to disconnect you from the game. Also, it has a 15 minute cooldown. You can disable the GetAll button via TSM's AuctionDB options if this feature doesn't work well on your server."] = ""
 -- L["Any items in the AuctionDB database that contain the search phrase in their names will be displayed."] = ""
 -- L["Are you sure you want to clear your AuctionDB data?"] = ""
 -- L["Ascending"] = ""

TradeSkillMaster_AuctionDB/Locale/frFR.lua

@@ -14,7 +14,7 @@ local L = LibStub("AceLocale-3.0"):NewLocale("TradeSkillMaster_AuctionDB", "frFR
 if not L then return end
 
 L["A full auction house scan will scan every item on the auction house but is far slower than a GetAll scan. Expect this scan to take several minutes or longer."] = "Une analyse complète de l'Hôtel des ventes examinera tous les objets de l'hôtel des ventes mais est beaucoup plus lente que la méthode du GetAll. Attendez-vous à ce que cette analyse dure de nombreuses minutes."
--- L["A GetAll scan is the fastest in-game method for scanning every item on the auction house. However, there are many possible bugs on Blizzard's end with it including the chance for it to disconnect you from the game. Also, it has a 15 minute cooldown."] = ""
+-- L["A GetAll scan is the fastest in-game method for scanning every item on the auction house. However, there are many possible bugs on Blizzard's end with it including the chance for it to disconnect you from the game. Also, it has a 15 minute cooldown. You can disable the GetAll button via TSM's AuctionDB options if this feature doesn't work well on your server."] = ""
 L["Any items in the AuctionDB database that contain the search phrase in their names will be displayed."] = "Tout objets présent dans la base de donnée d'AuctionDB contenant la phrase recherchée dans son nom sera affiché."
 L["Are you sure you want to clear your AuctionDB data?"] = "Êtes-vous sûr de vouloir vider les données d'AuctionDB?"
 L["Ascending"] = "Croissant"

TradeSkillMaster_AuctionDB/Locale/koKR.lua

@@ -14,7 +14,7 @@ local L = LibStub("AceLocale-3.0"):NewLocale("TradeSkillMaster_AuctionDB", "koKR
 if not L then return end
 
 L["A full auction house scan will scan every item on the auction house but is far slower than a GetAll scan. Expect this scan to take several minutes or longer."] = "전체 검색은 경매장 내의 모든 아이템을 검색하지만 GetAll 검색보다는 훨씬 느립니다. 이 검색은 몇 분 정도 또는 그 이상의 시간이 소요됩니다."
-L["A GetAll scan is the fastest in-game method for scanning every item on the auction house. However, there are many possible bugs on Blizzard's end with it including the chance for it to disconnect you from the game. Also, it has a 15 minute cooldown."] = "GetAll 검색은 게임 내에서 경매장의 모든 아이템을 검색하기 위한 가장 빠른 검색 방법입니다. 하지만 블리자드 쪽에 많은 버그가 존재하며 게임의 접속이 끊길 가능성도 있습니다. 또한, 15분의 쿨다운이 존재합니다." -- Needs review
+L["A GetAll scan is the fastest in-game method for scanning every item on the auction house. However, there are many possible bugs on Blizzard's end with it including the chance for it to disconnect you from the game. Also, it has a 15 minute cooldown. You can disable the GetAll button via TSM's AuctionDB options if this feature doesn't work well on your server."] = "GetAll 검색은 게임 내에서 경매장의 모든 아이템을 검색하기 위한 가장 빠른 검색 방법입니다. 하지만 블리자드 쪽에 많은 버그가 존재하며 게임의 접속이 끊길 가능성도 있습니다. 또한, 15분의 쿨다운이 존재합니다. You can disable the GetAll button via TSM's AuctionDB options if this feature doesn't work well on your server." -- Needs review
 L["Any items in the AuctionDB database that contain the search phrase in their names will be displayed."] = "이름에 검색 구문을 포함하는 AuctionDB 데이터베이스 내의 모든 아이템이 표시됩니다."
 L["Are you sure you want to clear your AuctionDB data?"] = "모든 AuctionDB 데이터를 삭제 하시겠습니까?"
 L["Ascending"] = "오름차순"

TradeSkillMaster_AuctionDB/Locale/ptBR.lua

@@ -14,7 +14,7 @@ local L = LibStub("AceLocale-3.0"):NewLocale("TradeSkillMaster_AuctionDB", "ptBR
 if not L then return end
 
 L["A full auction house scan will scan every item on the auction house but is far slower than a GetAll scan. Expect this scan to take several minutes or longer."] = "Um escaneamento completo da casa de leilões irá escanear todos os itens da casa de leilões, porém é bem mais lento que um escaneamento PegaTudo. Espere que este escaneamento demore vários minutos ou mais."
--- L["A GetAll scan is the fastest in-game method for scanning every item on the auction house. However, there are many possible bugs on Blizzard's end with it including the chance for it to disconnect you from the game. Also, it has a 15 minute cooldown."] = ""
+-- L["A GetAll scan is the fastest in-game method for scanning every item on the auction house. However, there are many possible bugs on Blizzard's end with it including the chance for it to disconnect you from the game. Also, it has a 15 minute cooldown. You can disable the GetAll button via TSM's AuctionDB options if this feature doesn't work well on your server."] = ""
 L["Any items in the AuctionDB database that contain the search phrase in their names will be displayed."] = "Qualquer item no bando de dados do AuctionDB que contém a frase procurada em seus nomes serão exibidos."
 L["Are you sure you want to clear your AuctionDB data?"] = "Você tem certeza de que quer limpar os dados do seu AuctionDB?"
 L["Ascending"] = "Crescente"

TradeSkillMaster_AuctionDB/Locale/ruRU.lua

@@ -14,7 +14,7 @@ local L = LibStub("AceLocale-3.0"):NewLocale("TradeSkillMaster_AuctionDB", "ruRU
 if not L then return end
 
 L["A full auction house scan will scan every item on the auction house but is far slower than a GetAll scan. Expect this scan to take several minutes or longer."] = "Полный скан Аукциона просканирует каждый товар, но намного дольше, чем GetAll скан. Ждите, это займёт несколько минут или более."
-L["A GetAll scan is the fastest in-game method for scanning every item on the auction house. However, there are many possible bugs on Blizzard's end with it including the chance for it to disconnect you from the game. Also, it has a 15 minute cooldown."] = "GetAll скан - самый быстрый внутриигровой способ сканирования. Однако, из-за из-за ошибок со стороны Blizzard's, существует вероятность отключения от сервера. Кроме того, он имеет 15-минутный перерыв."
+L["A GetAll scan is the fastest in-game method for scanning every item on the auction house. However, there are many possible bugs on Blizzard's end with it including the chance for it to disconnect you from the game. Also, it has a 15 minute cooldown. You can disable the GetAll button via TSM's AuctionDB options if this feature doesn't work well on your server."] = "GetAll скан - самый быстрый внутриигровой способ сканирования. Однако, из-за из-за ошибок со стороны Blizzard's, существует вероятность отключения от сервера. Кроме того, он имеет 15-минутный перерыв. You can disable the GetAll button via TSM's AuctionDB options if this feature doesn't work well on your server."  -- Needs review
 L["Any items in the AuctionDB database that contain the search phrase in their names will be displayed."] = "Будут отображены все товары из базы данных AuctionDB, содержащие искомую фразу в названии."
 L["Are you sure you want to clear your AuctionDB data?"] = "Вы действительно хотите очистить базу AuctionDB?"
 L["Ascending"] = "Возрастание"

TradeSkillMaster_AuctionDB/Locale/zhCN.lua

@@ -14,7 +14,7 @@ local L = LibStub("AceLocale-3.0"):NewLocale("TradeSkillMaster_AuctionDB", "zhCN
 if not L then return end
 
 L["A full auction house scan will scan every item on the auction house but is far slower than a GetAll scan. Expect this scan to take several minutes or longer."] = "完整扫描拍卖行内的所有物品，此方式远慢于快速扫描，预计费时几分钟甚至更久。"
-L["A GetAll scan is the fastest in-game method for scanning every item on the auction house. However, there are many possible bugs on Blizzard's end with it including the chance for it to disconnect you from the game. Also, it has a 15 minute cooldown."] = "快速扫描时扫描拍卖行中每件物品最快的方式。然而,在服务器端有着可能的BUG会使您掉线,所以每15分钟才能执行一次。"
+L["A GetAll scan is the fastest in-game method for scanning every item on the auction house. However, there are many possible bugs on Blizzard's end with it including the chance for it to disconnect you from the game. Also, it has a 15 minute cooldown. You can disable the GetAll button via TSM's AuctionDB options if this feature doesn't work well on your server."] = "快速扫描时扫描拍卖行中每件物品最快的方式。然而,在服务器端有着可能的BUG会使您掉线,所以每15分钟才能执行一次。 You can disable the GetAll button via TSM's AuctionDB options if this feature doesn't work well on your server."  -- Needs review
 L["Any items in the AuctionDB database that contain the search phrase in their names will be displayed."] = "任何包含搜索短语的AuctionDB数据库中的物品都将显示。"
 L["Are you sure you want to clear your AuctionDB data?"] = "您确定要清除AuctionDB数据吗?"
 L["Ascending"] = "升序"

TradeSkillMaster_AuctionDB/Locale/zhTW.lua

@@ -14,7 +14,7 @@ local L = LibStub("AceLocale-3.0"):NewLocale("TradeSkillMaster_AuctionDB", "zhTW
 if not L then return end
 
 L["A full auction house scan will scan every item on the auction house but is far slower than a GetAll scan. Expect this scan to take several minutes or longer."] = "完整的拍賣行掃描會掃描每件在拍賣行的物品但是比GetAll掃描還緩慢。預期掃描會花費幾分鐘或是更久。"
--- L["A GetAll scan is the fastest in-game method for scanning every item on the auction house. However, there are many possible bugs on Blizzard's end with it including the chance for it to disconnect you from the game. Also, it has a 15 minute cooldown."] = ""
+-- L["A GetAll scan is the fastest in-game method for scanning every item on the auction house. However, there are many possible bugs on Blizzard's end with it including the chance for it to disconnect you from the game. Also, it has a 15 minute cooldown. You can disable the GetAll button via TSM's AuctionDB options if this feature doesn't work well on your server."] = ""
 L["Any items in the AuctionDB database that contain the search phrase in their names will be displayed."] = "任何在AuctionDB資料庫中符合搜尋條件的物品，都會顯示出來。"
 L["Are you sure you want to clear your AuctionDB data?"] = "你確定要清除AuctionDB資料？"
 L["Ascending"] = "遞增"

TradeSkillMaster_AuctionDB/Modules/GUI.lua

@@ -51,12 +51,22 @@ function private:CreateStartScanContent(parent)
 	frame:SetAllPoints(parent)
 	frame:Hide()
 
+	-- Don't create or handle the GetAll button if player has disabled GetAll.
+	-- NOTE: This GUI creation is only done once per game reload, so this choice
+	-- won't change until the user does a UI "/reload" or logs out of the game.
+	local includeGetAll = not TSM.db.profile.disableGetAll
+
 	local function UpdateGetAllButton()
+		if not frame.startGetAllButton then
+			return  -- Do nothing if the GetAll feature is disabled.
+		end
+
 		if TSM.Scan.isScanning then
 			frame:Disable()
 		elseif not select(2, CanSendAuctionQuery()) then
+			-- Server says that GetAll isn't ready. Check our stored cooldown value.
 			local previous = TSM.db.profile.lastGetAll or time()
-			if previous > (time() - 15*60) then
+			if previous > (time() - 15*60) then  -- 15 minute enforced cooldown between GetAll scans...
 				local diff = previous + 15*60 - time()
 				local diffMin = math.floor(diff/60)
 				local diffSec = diff - diffMin*60
@@ -67,33 +77,36 @@ function private:CreateStartScanContent(parent)
 			frame:Enable()
 			frame.startGetAllButton:Disable()
 		else
+			-- Server says that GetAll is ready.
 			frame:Enable()
 			frame.getAllStatusText:SetText("|cff009900"..L["Ready"])
 			frame.startGetAllButton:Enable()
 		end
 	end
-	
-	frame:SetScript("OnShow", function(self)
+
+	if includeGetAll then
+		frame:SetScript("OnShow", function(self)
 			TSMAPI:CreateTimeDelay("auctionDBGetAllStatus", 0, UpdateGetAllButton, 0.2)
 		end)
-	
-	frame:SetScript("OnHide", function(self)
+
+		frame:SetScript("OnHide", function(self)
 			TSMAPI:CancelFrame("auctionDBGetAllStatus")
 		end)
-		
+	end
+
 	frame.Enable = function(self)
-		self.startGetAllButton:Enable()
+		if self.startGetAllButton then self.startGetAllButton:Enable() end
 		self.startFullScanButton:Enable()
 		self.startGroupScanButton:Enable()
 	end
 
 	frame.Disable = function(self)
-		self.startGetAllButton:Disable()
+		if self.startGetAllButton then self.startGetAllButton:Disable() end
 		self.startFullScanButton:Disable()
 		self.startGroupScanButton:Disable()
 	end
-	
-	-- top row (auto updater)
+
+	-- Top row: Auto updater.
 	local text = TSMAPI.GUI:CreateLabel(frame)
 	text:SetFont(TSMAPI.Design:GetContentFont(), 24)
 	text:SetPoint("TOP", 0, -24)
@@ -107,69 +120,81 @@ function private:CreateStartScanContent(parent)
 	a1:SetDuration(.5)
 	ag:SetLooping("BOUNCE")
 	ag:Play()
-	
+
 	local content = CreateFrame("Frame", nil, frame)
 	content:SetAllPoints(parent.content)
 	TSMAPI.Design:SetFrameBackdropColor(content)
-	
-	-- group tree
+
+	-- Group tree.
 	local container = CreateFrame("Frame", nil, content)
 	container:SetPoint("TOPLEFT", 5, -35)
 	container:SetPoint("BOTTOMRIGHT", -205, 5)
 	TSMAPI.Design:SetFrameColor(container)
 	frame.groupTree = TSMAPI:CreateGroupTree(container, nil, "AuctionDB")
-	
+
 	local bar = TSMAPI.GUI:CreateVerticalLine(content, 0)
 	bar:ClearAllPoints()
 	bar:SetPoint("TOPRIGHT", -200, -30)
 	bar:SetPoint("BOTTOMRIGHT", -200, 0)
-	
+
 	local buttonFrame = CreateFrame("Frame", nil, content)
 	buttonFrame:SetPoint("TOPLEFT", content, "TOPRIGHT", -200, 0)
 	buttonFrame:SetPoint("BOTTOMRIGHT")
-	
-	-- first row (getall scan)
+
+	-- Row: GetAll Scan.
+	-- NOTE: We hide this button if the player has disabled GetAll scans.
+	local yOffset = -50
+	if includeGetAll then
+		local btn = TSMAPI.GUI:CreateButton(buttonFrame, 18)
+		btn:SetPoint("TOPLEFT", 6, yOffset)
+		btn:SetPoint("TOPRIGHT", -6, yOffset)
+		btn:SetHeight(22)
+		btn:SetScript("OnClick", TSM.Scan.StartGetAllScan)
+		btn:SetText(L["Run GetAll Scan"])
+		btn.tooltip = L["A GetAll scan is the fastest in-game method for scanning every item on the auction house. However, there are many possible bugs on Blizzard's end with it including the chance for it to disconnect you from the game. Also, it has a 15 minute cooldown. You can disable the GetAll button via TSM's AuctionDB options if this feature doesn't work well on your server."]
+		frame.startGetAllButton = btn
+
+		local text = TSMAPI.GUI:CreateLabel(buttonFrame)
+		text:SetPoint("TOPLEFT", btn, "BOTTOMLEFT", 0, -3)
+		text:SetPoint("TOPRIGHT", btn, "BOTTOMRIGHT", 0, -3)
+		text:SetHeight(16)
+		text:SetJustifyH("CENTER")
+		text:SetJustifyV("CENTER")
+		frame.getAllStatusText = text
+
+		yOffset = yOffset - 50
+
+		TSMAPI.GUI:CreateHorizontalLine(buttonFrame, yOffset)
+
+		yOffset = yOffset - 20
+	end
+
+	-- Row: Full Scan.
 	local btn = TSMAPI.GUI:CreateButton(buttonFrame, 18)
-	btn:SetPoint("TOPLEFT", 6, -50)
-	btn:SetPoint("TOPRIGHT", -6, -50)
-	btn:SetHeight(22)
-	btn:SetScript("OnClick", TSM.Scan.StartGetAllScan)
-	btn:SetText(L["Run GetAll Scan"])
-	btn.tooltip = L["A GetAll scan is the fastest in-game method for scanning every item on the auction house. However, there are many possible bugs on Blizzard's end with it including the chance for it to disconnect you from the game. Also, it has a 15 minute cooldown."]
-	frame.startGetAllButton = btn
-	
-	local text = TSMAPI.GUI:CreateLabel(buttonFrame)
-	text:SetPoint("TOPLEFT", btn, "BOTTOMLEFT", 0, -3)
-	text:SetPoint("TOPRIGHT", btn, "BOTTOMRIGHT", 0, -3)
-	text:SetHeight(16)
-	text:SetJustifyH("CENTER")
-	text:SetJustifyV("CENTER")
-	frame.getAllStatusText = text
-	
-	TSMAPI.GUI:CreateHorizontalLine(buttonFrame, -110)
-	
-	-- second row (full scan)
-	local btn = TSMAPI.GUI:CreateButton(buttonFrame, 18)
-	btn:SetPoint("TOPLEFT", 6, -150)
-	btn:SetPoint("TOPRIGHT", -6, -150)
+	btn:SetPoint("TOPLEFT", 6, yOffset)
+	btn:SetPoint("TOPRIGHT", -6, yOffset)
 	btn:SetHeight(22)
 	btn:SetScript("OnClick", TSM.Scan.StartFullScan)
 	btn:SetText(L["Run Full Scan"])
 	btn.tooltip = L["A full auction house scan will scan every item on the auction house but is far slower than a GetAll scan. Expect this scan to take several minutes or longer."]
 	frame.startFullScanButton = btn
-	
-	TSMAPI.GUI:CreateHorizontalLine(buttonFrame, -200)
-	
-	-- third row (group scan)
+
+	yOffset = yOffset - 40
+
+	TSMAPI.GUI:CreateHorizontalLine(buttonFrame, yOffset)
+
+	yOffset = yOffset - 20
+
+	-- Row: Group Scan.
 	local btn = TSMAPI.GUI:CreateButton(buttonFrame, 18)
-	btn:SetPoint("TOPLEFT", 6, -225)
-	btn:SetPoint("TOPRIGHT", -6, -225)
+	btn:SetPoint("TOPLEFT", 6, yOffset)
+	btn:SetPoint("TOPRIGHT", -6, yOffset)
 	btn:SetHeight(22)
 	btn:SetScript("OnClick", GUI.StartGroupScan)
 	btn:SetText(L["Scan Selected Groups"])
 	btn.tooltip = L["This will do a slow auction house scan of every item in the selected groups and update their AuctionDB prices. This may take several minutes."]
 	frame.startGroupScanButton = btn
-	
+
 	return frame
 end
 

TradeSkillMaster_AuctionDB/Modules/Scanning.lua

@@ -11,94 +11,389 @@ local TSM = select(2, ...)
 local Scan = TSM:NewModule("Scan", "AceEvent-3.0")
 local L = LibStub("AceLocale-3.0"):GetLocale("TradeSkillMaster_AuctionDB") -- loads the localization table
 
+Scan.groupScanStartTime = 0
 Scan.groupScanData = {}
 Scan.filterList = {}
 Scan.numFilters = 0
+Scan.fullScanStartTime = 0
+Scan.fullScanSecondsPerPage = -1
+Scan.fullScanCompleteElapsed = nil
+
+local verifyNewAlgorithm = false  -- DEVELOPERS: Set to "true" to validate and benchmark the new market data algorithm!
 
 
-local function ScanCallback(event, ...)
+local function FullScanCallback(event, ...)
 	if event == "SCAN_PAGE_UPDATE" then
+		-- We're running a "Full Scan" and have received an auction page.
+		-- NOTE: These normal per-page scans receive 50 items per page, and will
+		-- successfully download ALL auctions on private servers, thanks to pagination.
+		-- For example, while Warmane's "GetAll" only returns 55000 of 126559 auctions,
+		-- the regular "Full Scan" mode retrieves all 2532 pages of 50 items each,
+		-- meaning that it covers 126600 auctions (and therefore grabs them all)
+		-- in this example. Users should always prefer "Full Scan" when "GetAll" fails.
 		local page, total = ...
-		TSM.GUI:UpdateStatus(format(L["Scanning page %s/%s"], page, total), page*100/total)
+
+		-- Calculate the current page progress and the remainder as floating-point values.
+		local progress_float = page / total
+		local remaining_float = 1.0 - progress_float
+
+		-- Estimate the total scan time, based on a MIX of the average per-page so far,
+		-- and the previous scan's averages stored in the database (if available).
+		-- NOTE: This callback triggers after we RECEIVED "page", so we count "page" too.
+		-- NOTE: We don't do any "live" updates of the progress bar text. We only
+		-- update the text labels when we receive a page, which is very CPU-efficient.
+		local time_estimate_str = ""
+		if (page >= 1) and (total > page) then
+			-- Calculate how many seconds have elapsed per page-request so far.
+			-- NOTE: We time it via the less-precise "time()" function, which
+			-- bluntly returns whole seconds. The alternative would be to use
+			-- "debugprofilestop()", which has millisecond-precision, but breaks
+			-- if another addon calls "debugprofilestart()" (which resets that
+			-- timer back to zero). It doesn't really matter, since our "seconds
+			-- per page" is constantly re-calculated based on the latest "total
+			-- amount of whole seconds elapsed", so it doesn't accumulate any
+			-- rounding errors and gets more precise the more pages have been
+			-- downloaded (after ~10 pages, it's practically as accurate as the
+			-- debug-timer). We have to use this technique for safety!
+			-- NOTE: Most servers will gradually slow down the page requests
+			-- across the first 300 requests or so, which will become slower
+			-- and slower, which means that the initial time estimate will
+			-- grow until it settles on the correct time remainder. There's
+			-- nothing we can do to predict those gradual slowdowns / throttling,
+			-- which is why we're also storing the last scan's "final average"
+			-- in the database and using that for our subsequent scan estimates.
+			local seconds_elapsed = abs(time() - Scan.fullScanStartTime)
+			local seconds_per_page = seconds_elapsed / page
+			-- local pages_remaining = total - page  -- Not used for anything.
+
+			-- Remember our "real", unweighted value, for later DB storage.
+			Scan.fullScanSecondsPerPage = seconds_per_page
+
+			-- Calculate a smoothly weighted "seconds per page" value based on
+			-- a linear mix between the current "seconds per page" and the
+			-- stored "final seconds per page value" from our previous scan.
+			-- As we reach 100%, we'll use 100% of the current "real seconds
+			-- per page". But at 0%, we'll use the stored value instead.
+			-- Between that, we linearly fade the values so that we react
+			-- smoothly to changes in speed. This solves the issue that all
+			-- servers face, which is their gradual slowdown of page fetches,
+			-- where they start out very fast (such as 1.1 seconds per page),
+			-- but will have slowed down when you're at the end (such as 2.5 per
+			-- page). Typical server slowdown in speed is roughly linear, which
+			-- is why our linear blend between "current estimate" and "finished
+			-- estimate from previous scan" creates the most accurate results
+			-- we're able to get, given the server behavior. It should also
+			-- work perfectly on servers which don't follow this pattern, such
+			-- as if they have a perfectly linear time between all pages without
+			-- any throttling at all, in which case both estimates will basically
+			-- agree anyway (both the current and the saved value). This is the
+			-- best we can do with the facts of the game. A totally accurate
+			-- estimate is impossible, but we're as accurate as we can be.
+			-- NOTE: This estimate cannot be improved, since practically all
+			-- servers apply random throttling, have various loads and slowdowns
+			-- throughout the day, etc. This is the best we can do since the
+			-- actual speed depends on the server and is pretty unpredictable.
+			-- It would be like trying to predict "the total download-time of a
+			-- file that keeps fluctuating between fast and slow speeds". The
+			-- best we can do is estimate based on current and previous speeds.
+			local last_scan_seconds_per_page = TSM.db.factionrealm.lastScanSecondsPerPage
+			if last_scan_seconds_per_page and last_scan_seconds_per_page > 0 then
+				-- TSM:Print(format("Read from DB: %f (Our unweighted estimate: %f)", last_scan_seconds_per_page, seconds_per_page))  -- DEBUG
+				seconds_per_page = (seconds_per_page * progress_float) + (last_scan_seconds_per_page * remaining_float)
+				-- TSM:Print(format("New, weighted estimate: %f (Progress: %f / Remaining: %f)", seconds_per_page, progress_float, remaining_float))  -- DEBUG
+			else
+				-- TSM:Print(format("Nothing in DB yet (Our unweighted estimate: %f)", seconds_per_page))  -- DEBUG
+			end
+
+			-- Estimate the "total time" requirement for ALL pages, rounded to
+			-- the nearest whole second, at least 1 second.
+			-- NOTE: We calculate the total estimate instead of the "remaining
+			-- time", because servers tend to fluctuate constantly between slowly
+			-- and then quickly sending the pages, which means a "pages_remaining"
+			-- timer is hard to understand in terms of real time remaining, since
+			-- we might get 10 pages within a few seconds and count down their
+			-- "seconds per page" amounts much faster than natural time, and
+			-- then suddenly stall for 30 seconds without getting any pages.
+			-- So a "remaining time" estimate would not move naturally. Instead,
+			-- we use a constantly updating "total time estimate" which follows
+			-- the server performance beautifully and is easy to understand.
+			-- NOTE: Thanks to the linear blend between historical and current
+			-- server performance, our estimate is very accurate yet responsive.
+			-- NOTE: The total page count is able to change during AH scan, when
+			-- more auctions are added or removed, which further contributes to
+			-- the confusion if we would use a "time remaining" display instead,
+			-- but since we use a "total time" estimate the user instead smoothly
+			-- sees the total estimate change when the page count changes.
+			local seconds_total_estimate = max(1, floor((total * seconds_per_page) + 0.5))
+
+			-- Convert the "elapsed / estimated" seconds into hours, minutes and seconds.
+			time_estimate_str = format(" (%s / ~%s)", TSMAPI:FormatHMS(TSMAPI:SecondsToHMS(seconds_elapsed)), TSMAPI:FormatHMS(TSMAPI:SecondsToHMS(seconds_total_estimate)))
+		end
+
+		-- Calculate progress bar from 0-100%.
+		local progress_bar = min(100 * progress_float, 100)
+
+		-- Display the progress bar with the time estimate.
+		TSM.GUI:UpdateStatus(format(L["Scanning page %s/%s"], page, total) .. time_estimate_str, progress_bar)
 	elseif event == "SCAN_COMPLETE" then
+		-- The whole scan is complete, and wasn't interrupted by the player.
+
+		-- Store the final "seconds elapsed per page request" into the database.
+		-- NOTE: We only update it here after complete scans, to avoid poisoning
+		-- with incorrect, partial-scan estimates, since most servers heavily
+		-- slow down their page requests over time. The completed scan is the truth.
+		if Scan.fullScanSecondsPerPage > 0 then
+			TSM.db.factionrealm.lastScanSecondsPerPage = Scan.fullScanSecondsPerPage
+		end
+
+		-- Calculate how many seconds the completed "Full Scan" took.
+		-- NOTE: We must cache it in this external variable, because "Full Scans"
+		-- use a threading callback which calls "DoneScanning()" one more time,
+		-- so we preserve the value to still display it via that callback too.
+		Scan.fullScanCompleteElapsed = abs(time() - Scan.fullScanStartTime)
+
+		-- Now process all of the fetched auctions, and display the total time elapsed.
 		local data = ...
 		Scan:ProcessScanData(data)
-		Scan:DoneScanning()
-	elseif event == "INTERRUPTED" then
+		Scan:DoneScanning(Scan.fullScanCompleteElapsed)
+	elseif event == "SCAN_INTERRUPTED" or event == "INTERRUPTED" then
+		-- We've been interrupted by the Auction House closing.
+		-- NOTE: "SCAN_INTERRUPTED" is from LibAuctionScan-1.0, which isn't used
+		-- by TSM anymore, and "INTERRUPTED" is from "TSM/Auction/AuctionScanning.lua",
+		-- which is what this scanner uses nowadays.
 		Scan:DoneScanning()
 	end
 end
 
 function Scan.ProcessGetAllScan(self)
-	local temp = 0
+	-- Await the "AUCTION_ITEM_LIST_UPDATE" event with the results of the "get all" scan.
+	-- NOTE: This event won't be received if the server ignored/throttled the "get all" scan.
+	local time_start = time()
+	local progress_bar = 0
 	while true do
-		temp = min(temp + 1, 100)
-		self:Sleep(0.2)
-		if not Scan.isScanning then return end
-		if Scan.getAllLoaded then
+		-- NOTE: The fake progress bar takes 20 seconds to fill to 100% with this interval.
+		progress_bar = min(progress_bar + 1, 100)  -- Fake progress bar from 0-100%.
+		self:Sleep(0.2)  -- Update progress bar with this interval while waiting for server data.
+		if not Scan.isScanning then return end  -- Failed or aborted scan.
+		if Scan.getAllLoaded then  -- We have received the list of auctions via "AUCTION_ITEM_LIST_UPDATE" event!
 			break
 		end
-		TSM.GUI:UpdateStatus(L["Running query..."], nil, temp)
+
+		-- NOTE: If it has taken more than 30 seconds, we can assume that the server
+		-- didn't send any data and won't be replying, most likely due to hidden
+		-- throttling (happens on many private servers, if scanning too soon after another).
+		local time_elapsed = time() - time_start
+		if time_elapsed >= 30 then
+			-- If this happens, the user has most likely been throttled and should try again later.
+			TSM.GUI:UpdateStatus(L["Running query... Server not responding due to throttling? Try again later..."], nil, 100)
+		else
+			TSM.GUI:UpdateStatus(L["Running query..."], nil, progress_bar)
+		end
 	end
 
+	-- IMPORTANT: As explained in the "Scan:AUCTION_ITEM_LIST_UPDATE()" code
+	-- comments, we're allowing GetAll scans even when the server DOESN'T return
+	-- all auctions. For example, Warmane has chosen to limit GetAll results to
+	-- only 55k auctions (even though they usually have 120k+ auctions). This
+	-- means that the market data calculations will be missing data. We should
+	-- therefore warn the user and tell them to use "Full Scan" instead, when
+	-- they're playing on such "limited GetAll" servers. But we'll still allow
+	-- use of "GetAll", with a warning message for people who INSIST on using it,
+	-- since even when it only looks at half the auctions, it can still give a
+	-- "pretty decent" idea of market values.
+	-- NOTE: It WOULD be "best" to always throw away (ignore) incomplete "GetAll"
+	-- data, but most casual players will prefer to have this feature even if those
+	-- incomplete scans are much less accurate than a proper "Full Scan".
+	-- NOTE: The inaccuracy of an incomplete "GetAll" scan completely depends on
+	-- how the server implements "GetAll". If the server sorts the cheapest
+	-- auctions first, then it's somewhat acceptable for popular items which
+	-- have many stacks (and are therefore likely to contain enough data points
+	-- in the "GetAll" result), and those items will "just" tend to be undervalued
+	-- by 2-10% less than their real market value (since their percentile-based
+	-- scans will look at less auctions than the real amount). But for very rare
+	-- or unpopular items, the partial "GetAll" scan is very dangerous, since
+	-- you might only receive the massively overpriced auctions of a certain
+	-- item, and thereby calculate an insanely high market value for it. These
+	-- dangers are increased if the server sends the auctions in a totally
+	-- random order, which means an even greater risk of market price pollution
+	-- by only seeing overpriced auctions for many of the items.
+	-- NOTE: Furthermore, receiving incomplete "GetAll" results means that TSM
+	-- will wipe all of its "cheapest, current buyout price" data for all items,
+	-- and then fills them with incorrect data (or nothing at all if an item
+	-- wasn't seen in the latest fetch), thus hindering your ability to look up
+	-- the correct "best current buyout prices" too.
+	-- NOTE: In summary, you'll have to use partial "GetAll" results at your
+	-- own risk! A "Full Scan" is ALWAYS much better when your server's "GetAll"
+	-- doesn't provide all auction data!
+	local shown, total = GetNumAuctionItems("list")
+	if total ~= Scan.getAllLoaded then  -- getAllLoaded = Same as "shown", but cached via our GetAll event.
+		-- NOTE: Message is not localized, because who the hell is gonna provide
+		-- translations to this project? It'd be a waste of time to translate it,
+		-- especially since we dynamically insert a word based on the result.
+		TSM:Print(format(
+			"WARNING: Your server has %d auctions, but it%s sent %d auctions to us. Please use the normal \"Full Scan\" instead, if you want to accurately calculate the real market values of items. Your server's \"GetAll\" scan doesn't fetch all auctions!",
+			total, ((total > shown) and " only" or ""), shown
+		))
+	else
+		TSM:Print(format("All auctions received from server (%d auctions)...", Scan.getAllLoaded))
+	end
+
+	-- Collect relevant data about the auctions that we've received.
+	-- NOTE: "Scan.getAllLoaded" is the count of auctions we've received from
+	-- the server. One per listing. We keep this cached value to constantly verify
+	-- that we're still looking at the same auction-list while we're processing.
 	local data = {}
-	for i=1, Scan.getAllLoaded do
-		TSM.GUI:UpdateStatus(format(L["Scanning page %s/%s"], 1, 1), i*100/Scan.getAllLoaded)
-		if i % 100 == 0 then
+	progress_bar = 0
+	for auction_idx=1, Scan.getAllLoaded do
+		if (auction_idx == 1) or (auction_idx == Scan.getAllLoaded) or ((auction_idx % 100) == 0) then
+			-- Update progress bar for the 1st, last, and every 100th auction.
+			progress_bar = min(100*(auction_idx/Scan.getAllLoaded), 100)  -- Calculate progress bar from 0-100%.
+			TSM.GUI:UpdateStatus(format(L["Scanning page %s/%s"], 1, 1), progress_bar)
+
+			-- Yield the CPU every 100th "auction listing", to prevent freezing the game.
 			self:Yield()
+
+			-- Verify the currently visible auction count, to ensure that we're
+			-- still analyzing the same list of "GetAll" auction items.
 			if GetNumAuctionItems("list") ~= Scan.getAllLoaded then
+				-- This can happen due to server/game issues, or if the user
+				-- closes the auction GUI while we're processing. If that
+				-- happens, we'll abort the scan without processing the data.
 				--TSM:Print(L["GetAll scan did not run successfully due to issues on Blizzard's end. Using the TSM application for your scans is recommended."])
-				TSM:Print("GetAll scan did not run successfully.")
-				Scan:DoneScanning()
+				TSM:Print("GetAll: Scan failed or aborted by user.")
+				Scan:DoneScanning()  -- Sets isScanning and getAllLoaded to nil.
 				return
 			end
 		end
 
-		local itemID = TSMAPI:GetItemID(GetAuctionItemLink("list", i))
-		--local _, _, count, _, _, _, _, _, _, buyout = GetAuctionItemInfo("list", i)
-		local _, _, count, _, _, _, _, _, buyout = GetAuctionItemInfo("list", i)
-		if itemID and buyout and buyout > 0 then
-			data[itemID] = data[itemID] or {records={}, minBuyout=math.huge, quantity=0}
-			data[itemID].minBuyout = min(data[itemID].minBuyout, floor(buyout/count))
-			data[itemID].quantity = data[itemID].quantity + count
-			for j=1, count do
-				tinsert(data[itemID].records, floor(buyout/count))
+		-- Retrieve information about this auction.
+		-- * itemID = The numeric ID of the item (such as 52021, which would represent "Iceblade Arrow" for example).
+		-- * stack_size = How many are in this auction's exact "stack" (such as 940, if they're selling a stack of 940).
+		-- * buyout = The buyout price of the auction (to get the per-item price, we need to divide "buyout / stack_size").
+		local itemID = TSMAPI:GetItemID(GetAuctionItemLink("list", auction_idx))
+		--local _, _, stack_size, _, _, _, _, _, _, buyout = GetAuctionItemInfo("list", auction_idx)
+		local _, _, stack_size, _, _, _, _, _, buyout = GetAuctionItemInfo("list", auction_idx)
+
+		-- Only process this auction if we saw a valid buyout price (ignore bid-only auctions, etc).
+		if itemID and buyout and (buyout > 0) then
+			-- Calculate the price per item, always rounded downwards.
+			-- NOTE: TSM's "buyout per item" calculations are actually a freaking
+			-- mess. They basically use "floor(buyout/count)" everywhere in the
+			-- code, EXCEPT in SOME places where they use "floor(buyout/count + 0.5)"
+			-- which means that those round to the nearest number instead, which
+			-- is definitely more correct. In fact, it would probably be even more
+			-- correct to enforce a minimum value of "1", otherwise a stack of
+			-- 100 items for a total buyout price of 10 copper would end up as
+			-- "0 copper per item" by the basic flooring algorithm (floor(0.1) = 0).
+			-- But whatever, it's extremely inconsistently used everywhere in TSM's
+			-- codebase, and most places use the plain "floor downwards", so we'll
+			-- do that too. It would be way too much effort to rewrite the HUNDREDS of
+			-- other code locations that deal with money (all via differently named
+			-- variables), just to handle smarter rounding, and it would require
+			-- a lot of effort to ensure that TSM's algorithms still work afterwards.
+			-- But yeah, just realize this: TSM's per-item calculations aren't great,
+			-- however it really DOESN'T MATTER MUCH since it only affects items
+			-- whose prices are less than 1 copper per item, which is probably
+			-- why TSM's author never noticed any issues with the basic "floor()".
+			local buyout_per_item = floor(buyout / stack_size)
+
+			-- Append to the existing "data to process" for this item ID if exists, else create new item.
+			data[itemID] = data[itemID]
+			if not data[itemID] then
+				data[itemID] = {records={}, minBuyout=math.huge, quantity=0}
 			end
+
+			-- Calculate the lowest "per-item buyout price" we're seeing for this item.
+			data[itemID].minBuyout = min(data[itemID].minBuyout, buyout_per_item)
+
+			-- Count the total amount of this item that exists on the auction house (adds together all stacks).
+			-- NOTE: This is super useful when we're calculating the market value in data.lua,
+			-- since it tells us immediately what the total count (quantity) of all new records is.
+			data[itemID].quantity = data[itemID].quantity + stack_size
+
+			-- BRAINDEAD OLD TSM CODE WHICH ADDS 1 RECORD PER ITEM IN THE STACK,
+			-- MEANING 500 STACKS OF 1000 ARROWS WOULD BE HALF A MILLION TABLE
+			-- ROWS AND WOULD LEAD TO "OUT OF MEMORY" ERRORS. DON'T DO THIS!
+			-- for j=1, stack_size do
+			-- 	tinsert(data[itemID].records, buyout_per_item)
+			-- end
+
+			-- Rewritten, intelligent code which adds 1 record per "stack" (auction) instead.
+			-- NOTE: We avoid using hash-keys, saving memory by using a numeric array instead.
+			-- NOTE: We'll store 1 table per unique itemID, and 1 record per "auction"
+			-- which passed our filters (has a buyout price), meaning that it's usually
+			-- a bit less than the total auction count on the server, and will never
+			-- be too many rows for WoW's Lua memory system to handle.
+			-- NOTE: If there is ever a server with so many auctions that even this
+			-- would reach WoW's Lua memory limits, then we could possibly store
+			-- the records as a concatenated, semicolon-separated string instead,
+			-- which would be slower but would handle millions of auctions with ease.
+			tinsert(data[itemID].records, {stack_size, buyout_per_item})
 		end
 	end
 
+	-- Process the collected "GetAll" auction data as a new "complete scan" with today's date.
 	TSM.db.factionrealm.lastCompleteScan = time()
-	TSM.Data:ProcessData(data)
+	TSM.Data:ProcessData(data, nil, verifyNewAlgorithm)
 
+	-- Show GUI progress while we're waiting for the processing.
+	-- NOTE: The status text will be set to "complete" elsewhere, automatically.
 	TSM.GUI:UpdateStatus(L["Processing data..."])
 	while TSM.processingData do
 		self:Sleep(0.2)
 	end
 
+	-- Processing is complete, so warn the user that they should reload the UI now.
 	TSM:Print(L["It is strongly recommended that you reload your ui (type '/reload') after running a GetAll scan. Otherwise, any other scans (Post/Cancel/Search/etc) will be much slower than normal."])
 end
 
 function Scan:AUCTION_ITEM_LIST_UPDATE()
 	Scan:UnregisterEvent("AUCTION_ITEM_LIST_UPDATE")
-	local num, total = GetNumAuctionItems("list")
 
-	--if num ~= total or num == 0 then
-	if num == 0 then
+	-- shown = How many auctions we received in the current data batch.
+	-- total = The total number of auction items.
+	local shown, total = GetNumAuctionItems("list")
+
+	-- OLD TSM CODE: It assumes that "GetAll" returns ALL auctions on official
+	-- Blizzard servers, so it also checks if "shown equals the total".
+	--if shown ~= total or shown == 0 then
+	-- WORKAROUND: Because places like Warmane with huge auction houses DON'T
+	-- return all items even via "GetAll" scans, we must ignore shown-vs-total
+	-- mismatches. The server won't let us query the subsequent pages since
+	-- there isn't any pagination in the "GetAll" API, and wouldn't work anyway
+	-- due to the 15-30 minute cooldown for "GetAll" calls, so we can't fetch
+	-- all auctions if the server is too popular. For example, typical
+	-- Warmane-Icecrown "GetAll" results will be: shown=55000, total=122523,
+	-- meaning that "GetAll" only receives about half of the total auctions.
+	if shown <= 0 then
 		--TSM:Print(L["GetAll scan did not run successfully due to issues on Blizzard's end. Using the TSM application for your scans is recommended."])
-		TSM:Print("GetAll scan did not run successfully.")
-		Scan:DoneScanning()
+		TSM:Print("GetAll: Scan failed due to server issues.")
+		Scan:DoneScanning()  -- Sets isScanning and getAllLoaded to nil.
 		return
 	end
-	Scan.getAllLoaded = num
+
+	-- Cache the amount of auctions we received on the current "page/batch", so that
+	-- we can validate that we're looking at this "page" while processing later.
+	Scan.getAllLoaded = shown
 end
 
 function Scan:GetAllScanQuery()
+	-- NOTE: This API doesn't work properly on some servers. For example, on Warmane,
+	-- it always claims that you can do a "GetAll" scan after you log in, even
+	-- if you're on cooldown and the server will actually be ignoring your request,
+	-- which is why we had to implement timeout detection in "ProcessGetAllScan()".
 	local canScan, canGetAll = CanSendAuctionQuery()
 	if not canGetAll then return TSM:Print(L["Can't run a GetAll scan right now."]) end
 	if not canScan then return TSMAPI:CreateTimeDelay(0.5, Scan.GetAllScanQuery) end
-	QueryAuctionItems("", nil, nil, nil, nil, nil, nil, nil, nil, true)
 	Scan:RegisterEvent("AUCTION_ITEM_LIST_UPDATE")
-	TSMAPI.Threading:Start(Scan.ProcessGetAllScan, 1, function() Scan:DoneScanning() end)
+	QueryAuctionItems("", nil, nil, nil, nil, nil, nil, nil, nil, true)
+	TSMAPI.Threading:Start(Scan.ProcessGetAllScan, 1, function()
+		-- Pass through the cached "full scan complete elapsed" value, which ONLY
+		-- contains a value if the latest full scan was successfully completed.
+		-- NOTE: This callback runs when the thread is finished, no matter what
+		-- reason. That's why we must preserve the "time elapsed" value for display.
+		Scan:DoneScanning(Scan.fullScanCompleteElapsed)
+	end)
 end
 
 local function GroupScanCallback(event, ...)
@@ -114,14 +409,23 @@ local function GroupScanCallback(event, ...)
 	elseif event == "QUERY_UPDATE" then
 		local current, total = ...
 		TSM.GUI:UpdateStatus(format(L["Preparing Filter %d / %d"], current, total))
-	elseif event == "SCAN_INTERRUPTED" then
+	elseif event == "SCAN_INTERRUPTED" or event == "INTERRUPTED" then
+		-- We've been interrupted by the Auction House closing.
+		-- NOTE: "SCAN_INTERRUPTED" is from LibAuctionScan-1.0, which isn't used
+		-- by TSM anymore, and "INTERRUPTED" is from "TSM/Auction/AuctionScanning.lua",
+		-- which is what this scanner uses nowadays.
 		Scan:DoneScanning()
 	elseif event == "SCAN_TIMEOUT" then
 		tremove(Scan.filterList, 1)
 		Scan:ScanNextGroupFilter()
 	elseif event == "SCAN_PAGE_UPDATE" then
 		local page, total = ...
-		TSM.GUI:UpdateStatus(format(L["Scanning %d / %d (Page %d / %d)"], Scan.numFilters-#Scan.filterList, Scan.numFilters, page+1, total), nil, page*100/total)
+		-- We have now received at least 1 page for this item. Show how many pages remain.
+		-- NOTE: We can't provide any time estimate here, since the other group sizes are unknown.
+		-- NOTE: We use this particular item's page-progress as the progress bar.
+		-- NOTE: We add "+1" to the page counter, to indicate that we've received that page and are working on the next page.
+		local progress_bar = min(100*(page/total), 100)  -- Calculate progress bar from 0-100%.
+		TSM.GUI:UpdateStatus(format(L["Scanning %d / %d (Page %d / %d)"], ((Scan.numFilters-#Scan.filterList) + 1), Scan.numFilters, min(page + 1, total), total), nil, progress_bar)
 	elseif event == "SCAN_COMPLETE" then
 		local data = ...
 		for _, itemString in ipairs(Scan.filterList[1].items) do
@@ -136,11 +440,23 @@ end
 
 function Scan:ScanNextGroupFilter(data)
 	if #Scan.filterList == 0 then
+		-- Calculate how many seconds the completed "Group Scan" took.
+		local seconds_elapsed = abs(time() - Scan.groupScanStartTime)
+
+		-- Now process all of the fetched auctions, and display the total time elapsed.
 		Scan:ProcessScanData(Scan.groupScanData)
-		Scan:DoneScanning()
+		Scan:DoneScanning(seconds_elapsed)
 		return
 	end
-	TSM.GUI:UpdateStatus(format(L["Scanning %d / %d (Page %d / %d)"], Scan.numFilters-#Scan.filterList, Scan.numFilters, 1, 1), (Scan.numFilters-#Scan.filterList)*100/Scan.numFilters)
+
+	-- Apply the temporary label for when we've requested the item's 1st page,
+	-- but we don't yet know how many results or pages there are for this item.
+	-- NOTE: We can't provide any time estimate here, since the other group sizes are unknown.
+	-- NOTE: In the label, we count the items starting at 1, to say "Scanning 1 / 2"
+	-- (instead of "Scanning 0 / 2"), but for the progress bar we count starting
+	-- from 0, so that it fills up properly by only proceeding after an item is done.
+	local progress_bar = min(100*((Scan.numFilters-#Scan.filterList)/Scan.numFilters), 100)  -- Calculate progress bar from 0-100%.
+	TSM.GUI:UpdateStatus(format(L["Scanning %d / %d (Page 1 / ?)"], ((Scan.numFilters-#Scan.filterList) + 1), Scan.numFilters), progress_bar)
 	TSMAPI.AuctionScan:RunQuery(Scan.filterList[1], GroupScanCallback)
 end
 
@@ -152,6 +468,7 @@ function Scan:StartGroupScan(items)
 	wipe(Scan.groupScanData)
 	Scan.numFilters = 0
 	TSMAPI.AuctionScan:StopScan()
+	Scan.groupScanStartTime = time()  -- Keep track of when we started the "Group Scan".
 	TSMAPI:GenerateQueries(items, GroupScanCallback)
 	TSM.GUI:UpdateStatus(L["Preparing Filters..."])
 end
@@ -162,10 +479,23 @@ function Scan:StartFullScan()
 	Scan.isBuggedGetAll = nil
 	Scan.groupItems = nil
 	TSMAPI.AuctionScan:StopScan()
-	TSMAPI.AuctionScan:RunQuery({name=""}, ScanCallback)
+	Scan.fullScanStartTime = time()  -- Keep track of when we started the "Full Scan".
+	Scan.fullScanSecondsPerPage = -1  -- Reset the page-speed timer.
+	Scan.fullScanCompleteElapsed = nil  -- Reset the "full scan completed" information.
+	TSMAPI.AuctionScan:RunQuery({name=""}, FullScanCallback)
 end
 
 function Scan:StartGetAllScan()
+	-- Refuse to perform "GetAll" if we're called while "GetAll" is disabled.
+	-- NOTE: Only happens if the player has visited the auction house and looked
+	-- at the AuctionDB GUI, and THEN gone into TSM's options to disable "GetAll".
+	-- The "Run GetAll Scan" button remains until /reload, so we must block it.
+	if TSM.db.profile.disableGetAll then
+		TSM:Print(L["You have disabled GetAll scans via AuctionDB's options."])
+		return
+	end
+
+	-- Begin the "GetAll" scan.
 	TSM.db.profile.lastGetAll = time()
 	Scan.isScanning = "GetAll"
 	Scan.isBuggedGetAll = nil
@@ -174,13 +504,21 @@ function Scan:StartGetAllScan()
 	Scan:GetAllScanQuery()
 end
 
-function Scan:DoneScanning()
-	TSM.GUI:UpdateStatus(L["Done Scanning"], 100)
+function Scan:DoneScanning(seconds_elapsed)
+	if seconds_elapsed then
+		-- If given the "time elapsed", display it as "Done Scanning (1:35:27)".
+		TSM.GUI:UpdateStatus(format("%s (%s)", L["Done Scanning"], TSMAPI:FormatHMS(TSMAPI:SecondsToHMS(seconds_elapsed))), 100)
+	else
+		-- Used when we don't care about showing time (such as scan failures).
+		TSM.GUI:UpdateStatus(L["Done Scanning"], 100)
+	end
 	Scan.isScanning = nil
 	Scan.getAllLoaded = nil
 end
 
 function Scan:ProcessScanData(scanData)
+	-- Handle scans performed via "Full Scan" and "Group Scan", but not "GetAll".
+	-- NOTE: See "Scan.ProcessGetAllScan()" for full explanation of this algorithm.
 	local data = {}
 
 	for itemString, obj in pairs(scanData) do
@@ -189,41 +527,97 @@ function Scan:ProcessScanData(scanData)
 			local quantity, minBuyout = 0, 0
 			local records = {}
 			for _, record in ipairs(obj.records) do
-				local itemBuyout = record:GetItemBuyout()
-				if itemBuyout and (itemBuyout < minBuyout or minBuyout == 0) then
-					minBuyout = itemBuyout
-				end
-				quantity = quantity + record.count
-				for i=1, record.count do
-					tinsert(records, itemBuyout)
+				-- Only process this auction if we saw a valid buyout price (ignore bid-only auctions, etc).
+				if record.buyout and record.buyout > 0 then
+					-- Calculate the price per item, always rounded downwards.
+					-- NOTE: "GetItemBuyout" returns nil if no buyout or if buyout is "0".
+					local itemBuyout = record:GetItemBuyout()
+					if itemBuyout then
+						-- Calculate the lowest "per-item buyout price" we're seeing for this item.
+						if (itemBuyout < minBuyout or minBuyout == 0) then
+							minBuyout = itemBuyout
+						end
+
+						-- Count the total amount of this item that exists on the auction house (adds together all stacks).
+						quantity = quantity + record.count
+
+						-- BRAINDEAD OLD TSM CODE WHICH ADDS 1 RECORD PER ITEM IN THE STACK,
+						-- MEANING 500 STACKS OF 1000 ARROWS WOULD BE HALF A MILLION TABLE
+						-- ROWS AND WOULD LEAD TO "OUT OF MEMORY" ERRORS. DON'T DO THIS!
+						-- for i=1, record.count do
+						-- 	tinsert(records, itemBuyout)
+						-- end
+
+						-- Rewritten, intelligent code which adds 1 record per "stack" (auction) instead.
+						tinsert(records, {record.count, itemBuyout})
+					end
 				end
 			end
+
+			-- Add this item to "data to process" even if there's zero records,
+			-- which can happen if they're all bid-only auctions.
+			-- NOTE: This differs from the behavior of "ProcessGetAllScan", which
+			-- only adds items that have at least 1 record with a buyout value.
+			-- NOTE: Empty records are totally fine either way, since "ProcessData"
+			-- simply ignores items that don't contain any buyout prices.
+			-- NOTE: If no buyout records were found, the "minBuyout" and "quantity"
+			-- fields below both have the default value of "0" (initialized above).
 			data[itemID] = {records=records, minBuyout=minBuyout, quantity=quantity}
 		end
 	end
 
+	-- Mark the collected auction data as a new "complete scan" with today's date,
+	-- but only if this was a normal "Full Scan" (not just a "TSM item group" scan).
 	if Scan.isScanning ~= "group" then
 		TSM.db.factionrealm.lastCompleteScan = time()
 	end
-	TSM.Data:ProcessData(data, Scan.groupItems)
+
+	-- Process the collected auction data.
+	TSM.Data:ProcessData(data, Scan.groupItems, verifyNewAlgorithm)
 end
 
 function Scan:ProcessImportedData(auctionData)
+	-- Handle manually imported auction scan data.
+	-- NOTE: This function is deprecated? Nothing seems to call it, unless they're
+	-- somehow calling it via another non-named technique, or perhaps it's internal
+	-- for developer-use only (basically just a quick way to emulate a full scan).
 	local data = {}
+
 	for itemID, auctions in pairs(auctionData) do
-		local quantity, minBuyout, records = 0, 0, {}
+		-- Process all imported auction records for this item.
+		local quantity, minBuyout = 0, 0
+		local records = {}
 		for _, auction in ipairs(auctions) do
+			-- Fetch the "price per item" and "item-count in this stack" from the auction's data.
+			-- NOTE: We only import auctions with per-item buyout values (ignore bid-only auctions, etc).
 			local itemBuyout, count = unpack(auction)
-			if itemBuyout and (itemBuyout < minBuyout or minBuyout == 0) then
-				minBuyout = itemBuyout
-			end
-			quantity = quantity + count
-			for i=1, count do
-				tinsert(records, itemBuyout)
+			if itemBuyout then
+				-- Calculate the lowest "per-item buyout price" we're seeing for this item.
+				if (itemBuyout < minBuyout or minBuyout == 0) then
+					minBuyout = itemBuyout
+				end
+
+				-- Count the total amount of this item that exists on the auction house (adds together all stacks).
+				quantity = quantity + count
+
+				-- BRAINDEAD OLD TSM CODE WHICH ADDS 1 RECORD PER ITEM IN THE STACK,
+				-- MEANING 500 STACKS OF 1000 ARROWS WOULD BE HALF A MILLION TABLE
+				-- ROWS AND WOULD LEAD TO "OUT OF MEMORY" ERRORS. DON'T DO THIS!
+				-- for i=1, count do
+				-- 	tinsert(records, itemBuyout)
+				-- end
+
+				-- Rewritten, intelligent code which adds 1 record per "stack" (auction) instead.
+				tinsert(records, {count, itemBuyout})
 			end
 		end
+
+		-- Add this item to "data to process" even if there's zero records,
+		-- which can happen if they're all bid-only auctions.
 		data[itemID] = {records=records, minBuyout=minBuyout, quantity=quantity}
 	end
+
+	-- Process the imported auction data as a new "complete scan" with today's date.
 	TSM.db.factionrealm.lastCompleteScan = time()
-	TSM.Data:ProcessData(data)
+	TSM.Data:ProcessData(data, nil, verifyNewAlgorithm)
 end

TradeSkillMaster_AuctionDB/Modules/config.lua

@@ -390,6 +390,13 @@ function Config:LoadOptions(container)
 							relativeWidth = 0.5,
 							tooltip = L["If checked, AuctionDB will add a tab to the AH to allow for in-game scans. If you are using the TSM app exclusively for your scans, you may want to hide it by unchecking this option. This option requires a reload to take effect."],
 						},
+						{
+							type = "CheckBox",
+							label = L["Disable \"GetAll\" Auction Scans"],
+							settingInfo = { TSM.db.profile, "disableGetAll" },
+							relativeWidth = 0.5,
+							tooltip = L["If checked, AuctionDB will not perform \"GetAll\" scans. This is useful if your server doesn't return all auctions in its \"GetAll\" results, which means that you'll get incorrect market value calculations for all items. If you're playing on such servers, it's best to disable the \"GetAll\" feature to avoid accidentally polluting your price database with incorrect data. This option takes effect immediately, but requires a reload to completely hide the \"Run GetAll Scan\" button."],
+						},
 					},
 				},
 				{

TradeSkillMaster_AuctionDB/Modules/data.lua

@@ -10,6 +10,21 @@
 local TSM = select(2, ...)
 local Data = TSM:NewModule("Data")
 
+-- locals to speed up function access
+local abs = abs
+local CopyTable = CopyTable
+local debugprofilestop = debugprofilestop
+local floor = floor
+local format = format
+local ipairs = ipairs
+local pairs = pairs
+local sqrt = sqrt
+local time = time
+local tinsert = tinsert
+local tsort = table.sort
+local type = type
+local unpack = unpack
+
 -- weight for the market value from X days ago (where X is the index of the table)
 local WEIGHTS = {[0] = 132, [1] = 125, [2] = 100, [3] = 75, [4] = 45, [5] = 34, [6] = 33,
 	[7] = 38, [8] = 28, [9] = 21, [10] = 15, [11] = 10, [12] = 7, [13] = 5, [14] = 4}
@@ -115,111 +130,540 @@ function Data:GetMarketValue(scans)
 	return totalWeight > 0 and floor(totalAmount / totalWeight + 0.5) or 0
 end
 
-function Data:ProcessData(scanData, groupItems)
-	if TSM.processingData then return TSMAPI:CreateTimeDelay(0.2, function() Data:ProcessData(scanData, groupItems) end) end
-	
+--- Process a table of new market scan data.
+-- @param scanData The market scan data.
+-- @param[opt] groupItems Affects how the minBuyout data is wiped. Use nil for regular behavior.
+-- @param[opt] verifyNewAlgorithm Boolean 'true' if you want to benchmark and verify the new market value algorithm.
+function Data:ProcessData(scanData, groupItems, verifyNewAlgorithm)
+	-- If we're currently processing data, retry in 0.2 seconds.
+	-- NOTE: This will retry itself over and over until it's able to process.
+	if TSM.processingData then
+		return TSMAPI:CreateTimeDelay(0.2, function() Data:ProcessData(scanData, groupItems, verifyNewAlgorithm) end)
+	end
 
-	-- wipe all the minBuyout data
+
+	-- Wipe all of our existing "minBuyout" data for the items included in the
+	-- new, incoming scan data in case of "Item Group scan", or for ALL currently
+	-- cached items in memory in other cases (such as "Full" and "GetAll" scans).
+	-- NOTE: It's no problem if we leave some items empty with "nil" minBuyout
+	-- values. That's how TSM is supposed to work, with items having an empty "minBuyout"
+	-- if there wasn't any "minBuyout" data for that item in the newest data batch.
 	if groupItems then
+		-- A list of items ("group scan") was provided. Wipe data for those items.
 		for itemString in pairs(groupItems) do
 			local itemID = TSMAPI:GetItemID(itemString)
-			if TSM.data[itemID] then
+			if TSM.data[itemID] then  -- If we have existing data for this item.
 				TSM:DecodeItemData(itemID)
-				TSM.data[itemID].minBuyout = nil
+				TSM.data[itemID].minBuyout = nil  -- Erase its stored minBuyout value.
 				TSM:EncodeItemData(itemID)
 			end
 		end
 	else
+		-- Wipe data for all items in memory, regardless of whether they're actually
+		-- included in the incoming scan data or not...
 		for itemID, data in pairs(TSM.data) do
 			TSM:DecodeItemData(itemID)
-			data.minBuyout = nil
+			data.minBuyout = nil  -- Directly updates TSM.data[itemID] via reference.
 			TSM:EncodeItemData(itemID)
 		end
 	end
-	
+
+
+	-- Convert the incoming "scanData" hashmap to a numerically indexed table,
+	-- to allow us to perform batched processing (since "pairs()" wouldn't know
+	-- how to resume at the current spot between our batch processing callbacks).
+	-- NOTE: Doesn't use much memory, since we're re-using "data" table refs.
 	local scanDataList = {}
 	for itemID, data in pairs(scanData) do
-		tinsert(scanDataList, {itemID, data})
+		scanDataList[#scanDataList + 1] = {itemID, data}
 	end
-	
-	-- go through each item and figure out the market value / update the data table
+
+
+	-- Go through each item and figure out their market value / update the data table.
+	-- NOTE: This processes the data in batched chunks of 500 items at a time,
+	-- pausing between each chunk to allow the game client to avoid freezing.
 	local index = 1
 	local day = Data:GetDay()
 	local function DoDataProcessing()
 		for i = 1, 500 do
+			-- Abort if we've reached the end of the processing queue.
 			if index > #scanDataList then
 				TSMAPI:CancelFrame("adbProcessDelay")
 				TSM.processingData = nil
 				break
 			end
-			
-			local itemID, data = unpack(scanDataList[index])		
-			TSM:DecodeItemData(itemID)
-			TSM.data[itemID] = TSM.data[itemID] or {scans={}, lastScan = 0}
-			local marketValue = Data:CalculateMarketValue(data.records)
-			
-			local scanData = TSM.data[itemID].scans
-			scanData[day] = scanData[day] or {avg=0, count=0}
-			if type(scanData[day]) == "number" then
-				-- this should never happen...
-				scanData[day] = {scanData[day]}
+
+			-- Detect which Item ID we're processing, and read its new data (new auction records).
+			local itemID, data = unpack(scanDataList[index])
+
+			-- Calculate the market value, and optionally perform benchmarks and
+			-- validation of the "new, compact algorithm" to prove correctness.
+			-- NOTE: We refuse to verify data with over 200 000 "item rows", since
+			-- that would risk bloating RAM and leading to a script crash. This
+			-- has been VERIFIED to still WORK with 169 000 rows, but fail for
+			-- an item with 686 900 table rows (script stops executing), so 200k
+			-- should be a safe cutoff to prevent memory overflows during benchmark.
+			local marketValue = -1
+			if (not verifyNewAlgorithm) or (data.quantity >= 200000) then
+				-- NOTE: Returns -1 if there aren't enough records to calculate.
+				marketValue = Data:CalculateMarketValue(data, verifyNewAlgorithm)
+			else
+				-- Verification and benchmark requested, and item is safe to check.
+
+				-- Perform the two calculations and benchmark the algorithms.
+				-- NOTE: Debug profiling is counted in milliseconds. For the old
+				-- algorithm, we're also including the time it takes to create
+				-- the "bloated table", since that's how the old TSM algorithm
+				-- created the table too (adds 1 row per "individual stack item").
+				-- SEE: https://wowpedia.fandom.com/wiki/API_debugprofilestop
+				local time_start_ms = debugprofilestop()
+
+				-- Generate an old-school data table, where we insert one row
+				-- per "item" per stack, so 5 stacks of 1000 items would mean
+				-- a total of 5000 rows, each with the individual item's price.
+				local data_new_records = data.records
+				local data_old_table = {records={}, minBuyout=data.minBuyout, quantity=data.quantity}
+				local data_old_records = data_old_table.records
+				for stack_idx=1, #data_new_records do
+					local stack_size, buyout_per_item = unpack(data_new_records[stack_idx])
+					for this_stack_item_idx=1, stack_size do
+						-- NOTE: The old algorithm used this exact tinsert()
+						-- method instead of the faster "tbl[#tbl + 1]" technique,
+						-- so that's why we're keeping that slow method here.
+						tinsert(data_old_records, buyout_per_item)
+					end
+				end
+
+				-- Verify that the "old-school table" contains ALL "expanded" items.
+				if #data_old_records ~= data.quantity then
+					TSM:Print(format("TABLE CREATION ERROR: item=%d, expected quantity=%d, created quantity=%d", itemID, data.quantity, #data_old_records))
+				end
+
+				-- Generate the old algorithm's value and finish its benchmark.
+				local old_marketValue = Data:CalculateMarketValue(data_old_table, verifyNewAlgorithm)
+				local time_elapsed_old_ms = (debugprofilestop() - time_start_ms)
+
+				-- Now generate the new algorithm's value and benchmark it too.
+				-- NOTE: This new algorithm is 1.3x-27.3x faster depending on
+				-- input data size, and on average 5x faster for most data. :)
+				time_start_ms = debugprofilestop()
+				marketValue = Data:CalculateMarketValue(data, verifyNewAlgorithm)
+				local time_elapsed_new_ms = (debugprofilestop() - time_start_ms)
+
+				-- Verify the calculations to ensure the algorithms are equal.
+				-- NOTE: Yes, the algorithms are perfectly equal for all input data.
+				if old_marketValue ~= marketValue then
+					TSM:Print(format("! ALGORITHM ERROR: item=%d, old=%.1f, new=%.1f", itemID, old_marketValue, marketValue))
+				end
+
+				-- Output benchmark results, but only for items with at least 500 entries.
+				-- NOTE: Comment this out if you're only interested in errors
+				-- above, or feel free to raise cutoff quantity to reduce logging.
+				if data.quantity >= 500 then
+					TSM:Print(format("+ ALGORITHM SPEED: item=%d, quantity=%d, match=%s, old=%.1f, new=%.1f, old_speed=%f ms, new_speed=%f ms, speedup=%.2f x",
+					itemID, data.quantity, old_marketValue == marketValue and "YES" or "NO", old_marketValue, marketValue, time_elapsed_old_ms, time_elapsed_new_ms, time_elapsed_old_ms > 0 and (time_elapsed_old_ms / time_elapsed_new_ms) or math.huge))  -- Prevents division by zero.
+				end
 			end
-			scanData[day].avg = scanData[day].avg or 0
-			scanData[day].count = scanData[day].count or 0
-			if #scanData[day] > 0 then
-				scanData[day] = Data:ConvertScansToAvg(scanData[day])
+
+			-- Detect whether it was POSSIBLE to calculate a market value, and
+			-- ONLY proceed with the item updates if we were able to calculate
+			-- a new market value. Otherwise, skip the item as if it "didn't even
+			-- exist in this scan", since it basically "doesn't exist" if there
+			-- were no buyout prices to calculate a new market value from!
+			-- NOTE: This can happen if the scan data only contained "bid without
+			-- buyout" items, meaning they didn't have any per-item buyout data,
+			-- which can ONLY happen via "Scanning.lua:ProcessScanData()" when
+			-- doing a normal "Full Scan" or "Group Scan" (not "GetAll"). If
+			-- there aren't any buyout prices for the item, it still gets added
+			-- without any "records". This differs from "GetAll" which only adds
+			-- items to the queue if they had at least one "buyout price" auction.
+			-- NOTE: We're skipping the empty/indeterminable items to ensure that
+			-- we have identical behavior for both "GetAll" and all other scan
+			-- types, so that we NEVER add "empty/missing" market values for items!
+			-- NOTE: We allow a market value of "0", since it means there was
+			-- valid data in the calculations. However, "0" is extremely unlikely
+			-- since it would require a single, huge stack of items for a price
+			-- of 1 copper or so, to make the per-item market value end up at
+			-- just "0" for that item. Basically, it's never gonna happen!
+			if marketValue and (marketValue >= 0) then
+				-- Fetch our archived data (if we have any) for this itemID.
+				TSM:DecodeItemData(itemID)
+				TSM.data[itemID] = TSM.data[itemID] or {scans={}, lastScan = 0}
+
+				-- Update market scan statistics for this item.
+				local scanData = TSM.data[itemID].scans
+				scanData[day] = scanData[day] or {avg=0, count=0}
+				if type(scanData[day]) == "number" then
+					-- Original code comment here: "This should never happen..."
+					-- NOTE: WTF was TSM's original author doing here? They're
+					-- converting "scanData[day]" into an array with 1 numeric
+					-- value, and mixing that array data with hashmap keys below,
+					-- so it seems like they're storing some data with numeric
+					-- keys and others with hashmap keys, all in the same table...
+					scanData[day] = {scanData[day]}
+				end
+				scanData[day].avg = scanData[day].avg or 0
+				scanData[day].count = scanData[day].count or 0
+				if #scanData[day] > 0 then
+					scanData[day] = Data:ConvertScansToAvg(scanData[day])
+				end
+				scanData[day].avg = floor((scanData[day].avg * scanData[day].count + marketValue) / (scanData[day].count + 1) + 0.5)
+				scanData[day].count = scanData[day].count + 1
+
+				-- Remember the item's scan date, cheapest buyout price on AH right now,
+				-- and how many items in total exist on AH (adds together all stacks).
+				-- NOTE: We only update "minBuyout" if the scanned data for that
+				-- item contains a "greater than 0" buyout value. That was mostly
+				-- necessary in the past, when TSM sloppily included bid-only items
+				-- in the data, but should no longer be able to happen with our new code!
+				TSM.data[itemID].lastScan = TSM.db.factionrealm.lastCompleteScan
+				TSM.data[itemID].minBuyout = data.minBuyout > 0 and data.minBuyout or nil
+				TSM.data[itemID].quantity = data.quantity  -- Counts all items of all stacks.
+				Data:UpdateMarketValue(TSM.data[itemID])
+
+				-- Update our archived, encoded representation of this item's data.
+				TSM:EncodeItemData(itemID)
 			end
-			scanData[day].avg = floor((scanData[day].avg * scanData[day].count + marketValue) / (scanData[day].count + 1) + 0.5)
-			scanData[day].count = scanData[day].count + 1
-			
-			TSM.data[itemID].lastScan = TSM.db.factionrealm.lastCompleteScan
-			TSM.data[itemID].minBuyout = data.minBuyout > 0 and data.minBuyout or nil
-			TSM.data[itemID].quantity = data.quantity
-			Data:UpdateMarketValue(TSM.data[itemID])
-			TSM:EncodeItemData(itemID)
-			
+
+			-- Update our processing-index to point at the next item.
 			index = index + 1
 		end
 	end
-	
+
 	TSM.processingData = true
 	TSMAPI:CreateTimeDelay("adbProcessDelay", 0, DoDataProcessing, 0.1)
 end
 
-function Data:CalculateMarketValue(records)
-	local totalNum, totalBuyout = 0, 0
-	local numRecords = #records
-	
-	for i=1, numRecords do
-		totalNum = i - 1
-		if i ~= 1 and i > numRecords*MIN_PERCENTILE and (i > numRecords*MAX_PERCENTILE or records[i] >= MAX_JUMP*records[i-1]) then
-			break
-		end
-		
-		totalBuyout = totalBuyout + records[i]
-		if i == numRecords then
-			totalNum = i
-		end
+--- Calculate the current market value of an item, from the given scan data.
+-- @param data The market scan data. Beware that we will automatically mutate the "data.records" table to sort the incoming data!
+-- @param[opt] hide_oldschool_warning Boolean 'true' to suppress the warning if you're using the old-school algorithm. This is only useful when benchmarking!
+function Data:CalculateMarketValue(data, hide_oldschool_warning)
+	-- All auctions/stacks for this item (contains their price per item, and each stack's item count).
+	-- NOTE: The old-school algorithm instead uses bloated records (see description further down).
+	local records = data.records
+
+	-- How many of this item currently exists in total on the auction house (combines all stacks).
+	-- NOTE: This is the sum of the per-stack counts of all "records", and can be trusted completely.
+	local total_quantity = data.quantity
+
+	-- If we've been given zero records, return a market value of -1 to signal the issue.
+	-- NOTE: If we don't do this filtering, we would end up with "division by zero" below.
+	if (type(records) ~= "table") or (#records <= 0) then
+		return -1
 	end
-	
-	local uncorrectedMean = totalBuyout / totalNum
-	local varience = 0
-	
-	for i=1, totalNum do
-		varience = varience + (records[i]-uncorrectedMean)^2
-	end
-	
-	local stdDev = sqrt(varience/totalNum)
-	local correctedTotalNum, correctedTotalBuyout = 1, uncorrectedMean
-	
-	for i=1, totalNum do
-		if abs(uncorrectedMean - records[i]) < 1.5*stdDev then
-			correctedTotalNum = correctedTotalNum + 1
-			correctedTotalBuyout = correctedTotalBuyout + records[i]
+
+
+	-- Determine which algorithm to use; either old-school or the smart, "compact" algorithm.
+	if type(records[1]) ~= "table" then
+		-- USE THE OLD, BRAINDEAD ALGORITHM IF WE'VE BEEN GIVEN OLD-SCHOOL "BLOATED" RECORDS.
+		-- NOTE: This old TSM algorithm relies on tables with millions of entries,
+		-- which often leads to out-of-memory crashes and is also extremely slow.
+		if not hide_oldschool_warning then
+			-- Warn if we've been called with old-school data and we haven't
+			-- been told to suppress this warning (benchmarks will suppress it).
+			TSM:Print("Warning: Calculating old-school market value. The calling code needs to be rewritten to use the new method!")
 		end
+
+		local totalNum, totalBuyout = 0, 0
+		local numRecords = #records
+
+		-- See "STEP 1" of new algorithm for explanation about why we MUST sort.
+		tsort(records, function(a_buyout_per_item, b_buyout_per_item)
+			-- Sort by "per-item buyout" in ascending order.
+			return a_buyout_per_item < b_buyout_per_item
+		end)
+
+		for i=1, numRecords do
+			totalNum = i - 1
+			if i ~= 1 and i > numRecords*MIN_PERCENTILE and (i > numRecords*MAX_PERCENTILE or records[i] >= MAX_JUMP*records[i-1]) then
+				break
+			end
+
+			totalBuyout = totalBuyout + records[i]
+			if i == numRecords then
+				totalNum = i
+			end
+		end
+
+		local uncorrectedMean = totalBuyout / totalNum
+		local variance = 0
+
+		for i=1, totalNum do
+			variance = variance + (records[i]-uncorrectedMean)^2
+		end
+
+		local stdDev = sqrt(variance/totalNum)
+		local correctedTotalNum, correctedTotalBuyout = 1, uncorrectedMean
+
+		for i=1, totalNum do
+			if abs(uncorrectedMean - records[i]) < 1.5*stdDev then
+				correctedTotalNum = correctedTotalNum + 1
+				correctedTotalBuyout = correctedTotalBuyout + records[i]
+			end
+		end
+
+		local correctedMean = floor(correctedTotalBuyout / correctedTotalNum + 0.5)
+
+		return correctedMean
+	else
+		-- Rewritten, cleaned up and faster algorithm, which uses almost zero memory
+		-- and NEVER causes any memory overflow crashes, unlike the old algorithm.
+		-- AUTHOR: Gnomezilla on Warmane-Icecrown [https://github.com/Bananaman].
+		-- NOTE: This new algorithm is 1.3x-27.3x faster depending on input data
+		-- size, and on average 5x faster for most data. :)
+		-- NOTE: All code is heavily commented, to help other programmers understand
+		-- the complex algorith, and to avoid future breakages due to misunderstandings.
+		-- NOTE: TSM's intended algorithm is also documented online, but they
+		-- describe a slightly altered (more modern) algorithm than what we're using:
+		-- https://support.tradeskillmaster.com/en_US/custom-strings/how-is-auctiondb-market-value-calculated
+		-- Archived in case TSM deletes the page: https://archive.ph/LhSOI
+
+
+		-- How many of the cheapest items to consider (default: at least
+		-- 15%, at most 30% of the cheapest items). All items which are more
+		-- expensive than them are ignored.
+		-- NOTE: This is considered as the total of all items (combined
+		-- quantity of all items in all stacks). So if the first (cheapest)
+		-- stack is massive, and subsequent stacks are small, then we'll
+		-- only be calculating the value of the items of the 1st stack.
+		local idx_min_percentile = total_quantity * MIN_PERCENTILE
+		local idx_max_percentile = total_quantity * MAX_PERCENTILE
+
+		-- Keep track of how many items we've processed and their combined buyout.
+		local processed_quantity, processed_total_buyout = 0, 0
+
+		-- Cutoff value for how much the "next auction" is allowed to cost,
+		-- so that we can ignore all overpriced items. Default: Any price
+		-- increase higher than 120% will discard all subsequent auctions.
+		-- NOTE: We only need to update this when we switch to another "stack",
+		-- and we're initializing it to a special "infinity" value (math.huge)
+		-- to ensure that we don't use this value until we've calculated it.
+		local max_jump_buyout_per_item = math.huge
+
+		-- Keep track of the total "item index" we're at while we're traversing
+		-- through our compact "stacks". This emulates the classic way TSM
+		-- keeps track of the item/record counter.
+		local item_idx = 0
+
+		-- Used for signaling that we want to abort processing the remaining records.
+		local skip_remaining_records = false
+
+
+		-- STEP 1 (EXTREMELY IMPORTANT): We CANNOT trust the input. Our market
+		-- value algorithm ONLY WORKS if the prices are SORTED in ASCENDING ORDER,
+		-- but the auctions themselves are in RANDOM ORDER by default, in most
+		-- cases. So we MUST forcibly sort them now, otherwise we'll randomly
+		-- end up with very expensive items at the start of the list, which then
+		-- becomes extremely INCORRECT market values in our database, such as
+		-- thinking that "Wool Cloth" could be worth crazy amounts like "50 gold
+		-- per 1 wool cloth" instead of its real market value, thus breaking TSM!
+		tsort(records, function(a, b)
+			-- NOTE: Direct table refs instead of unpack() speeds up the sorting
+			-- by 3x, due to all the calls/comparisons involved when sorting.
+			local a_stack_size, a_buyout_per_item = a[1], a[2]
+			local b_stack_size, b_buyout_per_item = b[1], b[2]
+
+			-- Sort by "per-item buyout" in ascending order, and use "stack size"
+			-- in ascending order as a fallback if two stacks have identical prices.
+			if a_buyout_per_item ~= b_buyout_per_item then
+				return a_buyout_per_item < b_buyout_per_item
+			else
+				return a_stack_size < b_stack_size
+			end
+		end)
+
+
+		-- STEP 2: Calculate the total buyout of all items we'll be processing.
+
+		-- Process all of the compact "stack" records.
+		-- NOTE: Every record is 1 auction/stack, with a size and buyout-per-item.
+		for stack_idx=1, #records do
+			-- Fetch the stack size and buyout-per-item of this stack.
+			local stack_size, buyout_per_item = unpack(records[stack_idx])
+
+			-- Run TSM's algorithm for EVERY individual "item" in the stack's size,
+			-- so that we perform the same per-"item" work as their braindead algo.
+			-- NOTE: Does nothing if "stack_size" is <= 0.
+			-- NOTE: Technically, it would be possible to further optimize this
+			-- algorithm to "skip past entire stacks" all at once, instead of
+			-- processing every individual "virtual item from the stack", but
+			-- the algorithm is already so fast that adding extra complexity
+			-- is pointless and would just make it much harder to maintain!
+			for this_stack_item_idx=1, stack_size do
+				-- Increment the "total item index" that we're at now (across all stacks).
+				-- NOTE: Emulates TSM's counting of every "item" as individual records.
+				item_idx = item_idx + 1
+
+				-- Calculate the max allowed "buyout-per-item" price jump of the
+				-- current item.
+				-- NOTE: Yeah this leads to weirdness if we ended up including items
+				-- that were overpriced but were required by the "must process 15%+
+				-- of all items no matter what" TSM requirement, but TSM fixes
+				-- that problem during later filtering "steps" of the algorithm.
+				-- NOTE: We're only recalculating the value when the "previous
+				-- item" changes, meaning that unlike the old-school TSM algo,
+				-- we don't waste time re-calculating it thousands of times in
+				-- a row for large stacks of identical "per-item" prices.
+				-- NOTE: This optimization has been verified to generate the EXACT
+				-- same behavior/result as the old-schoold TSM algorithm.
+				-- NOTE: Yes, item 1 of stack 1 keeps the pre-initialized value
+				-- of "math.huge (infinity)" (above), since we don't need it yet.
+				-- NOTE: In effect, we're ALWAYS comparing the CURRENT item's
+				-- buyout price to a "max buyout" calculated from the "PREVIOUS
+				-- item". This "stack magic" is just an optimization to reduce
+				-- the CPU usage of the algorithm!
+				if this_stack_item_idx == 1 and stack_idx >= 2 then
+					-- We're on the FIRST item of a NEW stack (not stack 1).
+					-- Re-calculate "max buyout price" based on PREVIOUS stack's price.
+					-- NOTE: We could cache the "previous" data to avoid the lookup
+					-- here, but we wouldn't really save any meaningful CPU usage
+					-- and it would be harder for other programmers to understand.
+					local previous_stack_size, previous_buyout_per_item = unpack(records[stack_idx - 1])
+					max_jump_buyout_per_item = MAX_JUMP * previous_buyout_per_item
+				elseif this_stack_item_idx == 2 then
+					-- We're on the SECOND item of SAME stack (any stack, even stack 1).
+					-- Re-calculate "max buyout price" based on THIS stack's price.
+					max_jump_buyout_per_item = MAX_JUMP * buyout_per_item
+				end
+
+				-- If we're on "total item index" 2 or higher, and we've processed
+				-- at least "min percentile" amount of records, AND we've -EITHER-
+				-- reached the "maximum percentile" too -OR- we've reached a severely
+				-- overpriced stack before we could reach the "maximum percentile",
+				-- and if so, we'll skip all other records (remaining stacks/items).
+				if item_idx >= 2
+				and item_idx > idx_min_percentile
+				and (item_idx > idx_max_percentile or buyout_per_item >= max_jump_buyout_per_item) then
+					-- Signal that we want to skip the remaining records.
+					skip_remaining_records = true
+					break
+				end
+
+				-- Keep track of the total buyout value of all processed items.
+				-- NOTE: This is the main purpose of this particular loop.
+				processed_total_buyout = processed_total_buyout + buyout_per_item
+
+				-- Calculate "how many items we have processed", which is simply
+				-- the same as the current item index.
+				-- NOTE: TSM's classic algo does this as "item_idx - 1" because
+				-- they update this counter BEFORE they've decided whether to
+				-- actually process the current item. We instead only update this
+				-- after we've processed and incremented "total buyout" above,
+				-- which has the same end result but is much easier to understand.
+				-- NOTE: TSM's classic algo also has a weird chunk after this line,
+				-- which does "if item_idx == total_quantity then
+				-- processed_quantity = item_idx; end", which is complete nonsense
+				-- and was only needed due to their braindead sequence of updating
+				-- the "processed_quantity" variable, since they clearly just tried
+				-- to avoid having "processed_quantity = 0" when "total_quantity = 1",
+				-- but that was an extremely idiotic "fix" for their bad code. :)
+				processed_quantity = item_idx
+			end
+
+			-- Exit from the nested loop if we've been told to stop processing records.
+			if skip_remaining_records then break end
+		end
+
+		--  TSM:Print(format("processed_quantity: %d, processed_total_buyout: %d", processed_quantity, processed_total_buyout))  -- DEBUG
+
+
+		-- STEP 3: Calculate the mean (simple average) of all processed items.
+
+		local uncorrected_mean = processed_total_buyout / processed_quantity
+
+
+		-- STEP 4: Calculate the standard deviation of all processed items.
+
+		local variance = 0
+
+		-- Process all of the compact "stack" records again, but stop when we hit the limit.
+		-- NOTE: To understand this looping algorithm, look at STEP 2's comments above.
+		item_idx = 0
+		skip_remaining_records = false
+		for stack_idx=1, #records do
+			local stack_size, buyout_per_item = unpack(records[stack_idx])
+			for this_stack_item_idx=1, stack_size do
+				item_idx = item_idx + 1
+
+				-- Process up to and including "processed_quantity", but not higher.
+				if item_idx > processed_quantity then
+					skip_remaining_records = true
+					break
+				end
+
+				-- Calculate the updated variance.
+				variance = variance + ((buyout_per_item - uncorrected_mean)^2)
+			end
+
+			if skip_remaining_records then break end
+		end
+
+		local std_dev = sqrt(variance / processed_quantity)
+
+		--  TSM:Print(format("std_dev: %f, variance: %f", std_dev, variance))  -- DEBUG
+
+
+		-- STEP 5: Ignore all data points that are more than 1.5x std_dev away from the average.
+
+		-- Initialize the "corrected" quantity and buyout with 1 "fake" item
+		-- that has the same value as the uncorrected mean.
+		-- NOTE: We're replicating TSM 2.8's classic algorithm here... even
+		-- though their algorithm might not be perfect.
+		local corrected_processed_quantity, corrected_processed_total_buyout = 1, uncorrected_mean
+
+		-- Calculate the standard deviation cutoff. Anything further away will be ignored.
+		local std_dev_cutoff = 1.5 * std_dev
+
+		-- Process all of the compact "stack" records again, but stop when we hit the limit.
+		-- NOTE: To understand this looping algorithm, look at STEP 2's comments above.
+		item_idx = 0
+		skip_remaining_records = false
+		for stack_idx=1, #records do
+			local stack_size, buyout_per_item = unpack(records[stack_idx])
+
+			-- Speedup: Since the "buyout_per_item" only changes when we switch
+			-- to a different stack (all items in a stack have the same per-item
+			-- prices), we can therefore pre-calculate their deviation from the
+			-- average (mean) here, to avoid having to do it per-item below.
+			local abs_deviation_from_mean = abs(uncorrected_mean - buyout_per_item)
+
+			-- Speedup: We can also pre-calculate whether the items of this "stack"
+			-- all fit the rule of "they're less than std_dev cutoff away from mean".
+			local stack_include_items = abs_deviation_from_mean < std_dev_cutoff
+
+			-- Loop through all virtual "items" of the current "stack".
+			for this_stack_item_idx=1, stack_size do
+				item_idx = item_idx + 1
+
+				-- Process up to and including "processed_quantity", but not higher.
+				if item_idx > processed_quantity then
+					skip_remaining_records = true
+					break
+				end
+
+				-- Calculate the filtered "quantity" and "total buyout", which
+				-- ignores anything that's more than "std_dev_cutoff" away from avg.
+				if stack_include_items then
+					corrected_processed_quantity = corrected_processed_quantity + 1
+					corrected_processed_total_buyout = corrected_processed_total_buyout + buyout_per_item
+				end
+			end
+
+			if skip_remaining_records then break end
+		end
+
+		--  TSM:Print(format("corrected_processed_quantity: %d, corrected_processed_total_buyout: %d", corrected_processed_quantity, corrected_processed_total_buyout))  -- DEBUG
+
+
+		-- STEP 6: Calculate our current market value by simply taking the
+		-- average of the remaining (filtered) data points.
+		-- NOTE: This method ensures that no poisoning of our market value can
+		-- take place by those who post high volume items at astronomical prices.
+		-- It also gets rid of more subtle outliers to determine the average.
+
+		local corrected_mean = floor((corrected_processed_total_buyout / corrected_processed_quantity) + 0.5)
+
+
+		return corrected_mean
 	end
-	
-	local correctedMean = floor(correctedTotalBuyout / correctedTotalNum + 0.5)
-	
-	return correctedMean
 end

TradeSkillMaster_AuctionDB/TradeSkillMaster_AuctionDB.lua

@@ -22,6 +22,7 @@ local savedDBDefaults = {
 		scanData = "",
 		time = 0,
 		lastCompleteScan = 0,
+		lastScanSecondsPerPage = -1,
 		appDataUpdate = 0,
 	},
 	profile = {
@@ -33,6 +34,7 @@ local savedDBDefaults = {
 		marketValueTooltip = true,
 		minBuyoutTooltip = true,
 		showAHTab = true,
+		disableGetAll = false,
 	},
 }
 
@@ -372,9 +374,10 @@ local function encodeScans(scans)
 	local tbl, tbl2 = {}, {}
 	for day, data in pairs(scans) do
 		if type(data) == "table" and data.count and data.avg then
+			-- New method of encoding scans.
 			data = encode(data.avg).."@"..encode(data.count)
 		elseif type(data) == "table" then
-			-- Old method of encoding scans
+			-- Old method of encoding scans.
 			for i = 1, #data do
 				tbl2[i] = encode(data[i])
 			end
@@ -394,37 +397,48 @@ local function decodeScans(rope)
 	local currentDay = TSM.Data:GetDay()
 	for _, data in ipairs(days) do
 		local day, marketValueData = (":"):split(data)
-		day = decode(day)
-		scans[day] = {}
-		
-		--bug fix? ...SkillMaster_AuctionDB\TradeSkillMaster_AuctionDB.lua:398: bad argument #1 to 'strfind' (string expected, got nil)
-		if marketValueData ~= nil then
-		
-			if strfind(marketValueData, "@") then
-				local avg, count = ("@"):split(marketValueData)
-				avg = decode(avg)
-				count = decode(count)
-				if avg ~= "~" and count ~= "~" then
-					if abs(currentDay - day) <= TSM.MAX_AVG_DAY then
-						scans[day].avg = avg
-						scans[day].count = count
-					else
-						scans[day] = avg
+		-- BUG FIXED: Guard against incorrectly encoded "day" or "marketValueData",
+		-- which can happen extremely rarely due to some very rare, random bug
+		-- somewhere else in TSM (or perhaps due to mixing different versions
+		-- of TSM data). The cause of the rare corruption hasn't been found.
+		-- NOTE: We simply skip any "days/market values" that cannot be decoded,
+		-- which thereby ensures that we get a cleaned-up "decode" of the data,
+		-- so that TSM will then write the fixed data when it next "re-encodes"
+		-- the "decoded in-memory representation" of this item's data!
+		if day ~= nil and day ~= "" and marketValueData ~= nil and marketValueData ~= "" then
+			day = decode(day)
+			-- BUG FIXED: Verify yet again that the day itself was properly decoded,
+			-- but this time only check for "nil" which indicates "decode()" failure.
+			if day ~= nil then
+				-- Create a "scans" table entry for the decoded day.
+				scans[day] = {}
+
+				if strfind(marketValueData, "@") then
+					-- New method of decoding scans.
+					local avg, count = ("@"):split(marketValueData)
+					avg = decode(avg)
+					count = decode(count)
+					if avg ~= "~" and count ~= "~" then
+						if abs(currentDay - day) <= TSM.MAX_AVG_DAY then
+							scans[day].avg = avg
+							scans[day].count = count
+						else
+							scans[day] = avg
+						end
 					end
-				end
-			else
-				-- Old method of decoding scans
-				for _, value in ipairs({(";"):split(marketValueData)}) do
-					local decodedValue = decode(value)
-					if decodedValue ~= "~" then
-						tinsert(scans[day], tonumber(decodedValue))
+				else
+					-- Old method of decoding scans.
+					for _, value in ipairs({(";"):split(marketValueData)}) do
+						local decodedValue = decode(value)
+						if decodedValue ~= "~" then
+							tinsert(scans[day], tonumber(decodedValue))
+						end
+					end
+					if day ~= currentDay then
+						scans[day] = TSM.Data:GetAverage(scans[day])
 					end
-				end
-				if day ~= currentDay then
-					scans[day] = TSM.Data:GetAverage(scans[day])
 				end
 			end
-			
 		end
 	end