Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
35 commits
Select commit Hold shift + click to select a range
892c15f
Catch but do not thread job execution
remeike Jun 11, 2022
9b81060
Merge pull request #2 from remeike/non-threaded-jobs
remeike Jun 12, 2022
a8d1f2b
Add batch ID and counters for batch job
remeike Nov 17, 2022
51736a2
Fix jobsFromQueue and kill unused threads in tests
remeike Nov 18, 2022
a4618a7
Add batch job type and tests
remeike Nov 18, 2022
7ce929f
Add callback
remeike Nov 18, 2022
ee2c9a4
Remove fdescribe
remeike Nov 18, 2022
cb3e5b1
Move batch functions into lua scripts
remeike Nov 18, 2022
f297329
Rename BatchJob to BatchSummary
remeike Nov 19, 2022
b0a6b8a
Remove redundant line of code
remeike Nov 19, 2022
3adca57
Create separate function for stop batch queueing
remeike Nov 19, 2022
b8ba9dd
Add test for large batch job
remeike Nov 19, 2022
965cbcc
Add expiration to batch
remeike Nov 19, 2022
54ba9f5
Finish batch with stop batch queuing when jobs are all run
remeike Nov 19, 2022
e61bb74
Make batch expiration optional
remeike Nov 23, 2022
ea9ac1d
Bump version
remeike Nov 23, 2022
1ca2372
Merge pull request #3 from remeike/batch-jobs
remeike Nov 23, 2022
72fb243
Change formatting
remeike Nov 24, 2022
803124f
Remove Aeson helper module
remeike Nov 24, 2022
a68f40b
Add StrictData and fix couple typos
remeike Nov 25, 2022
7fb6fb1
Merge pull request #4 from remeike/reformat
remeike Nov 29, 2022
5fe4115
Make queuing batched jobs atomic
remeike Dec 5, 2022
4eb0b0e
Clear up warnings
remeike Dec 12, 2022
8b2e98d
Merge pull request #5 from remeike/atomic-queueing
remeike Dec 12, 2022
e4e2b83
Add batch completed function to config type
remeike Dec 21, 2022
2a256fd
Merge pull request #6 from remeike/add-batch-callback-to-config
remeike Dec 21, 2022
5d90eb2
Add scheduled and recurring jobs
remeike Dec 30, 2022
b3f28cf
Remove recurring job code and modify job to take entire hworker as ar…
remeike Jan 2, 2023
ecc1e1c
Return more information when queueing fails
remeike Feb 27, 2023
3e5ee88
Add failed as a status
remeike Feb 27, 2023
8970b74
Short circuit queueing
remeike Mar 3, 2023
f108172
Separate streamBatch functions
remeike Mar 14, 2023
a6fa84d
Add documentation for streamBatch functions
remeike Mar 14, 2023
453e4b2
Merge pull request #7 from remeike/scheduled-jobs
remeike Jul 6, 2023
88e0b2d
Create separate scheduler thread
remeike Jul 13, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion .gitignore
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
.cabal-sandbox
.stack-work
cabal.sandbox.config
dist-stack
dist-stack
stack.yaml.lock
4 changes: 4 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,3 +1,7 @@
* 0.3.0 Remeike Forbes 2022-11-23

Introduce batched jobs

* 0.2.0 Remeike Forbes

Coerce jobs to string (to work with Redis 5)
Expand Down
74 changes: 47 additions & 27 deletions example/src/Main.hs
Original file line number Diff line number Diff line change
@@ -1,37 +1,57 @@
{-# LANGUAGE DeriveGeneric #-}
{-# LANGUAGE MultiParamTypeClasses #-}
{-# LANGUAGE OverloadedStrings #-}
import Control.Concurrent (forkIO, threadDelay)
import Control.Concurrent.MVar (MVar, newMVar, putMVar, takeMVar)
import Control.Monad (forever)
import Data.Aeson (FromJSON, ToJSON)
import qualified Data.Text as T
import GHC.Generics (Generic)


--------------------------------------------------------------------------------
import Control.Concurrent ( forkIO, threadDelay )
import Control.Concurrent.MVar ( MVar, newMVar, putMVar, takeMVar )
import Control.Monad ( forever )
import Data.Aeson ( FromJSON, ToJSON )
import GHC.Generics ( Generic )
--------------------------------------------------------------------------------
import System.Hworker
--------------------------------------------------------------------------------


data PrintJob
= PrintA
| PrintB
deriving (Generic, Show)


newtype State =
State (MVar Int)


data PrintJob = PrintA | PrintB deriving (Generic, Show)
data State = State (MVar Int)
instance ToJSON PrintJob
instance FromJSON PrintJob

loopForever :: a
loopForever = loopForever

instance Job State PrintJob where
job (State mvar) PrintA =
do v <- takeMVar mvar
if v == 0
then do putMVar mvar 0
putStrLn "A" >> return Success
else do putMVar mvar (v - 1)
error $ "Dying: " ++ show v

job _ PrintB = putStrLn "B" >> return Success

main = do mvar <- newMVar 3
hworker <- create "printer" (State mvar)
forkIO (worker hworker)
forkIO (monitor hworker)
forkIO (forever $ queue hworker PrintA >> threadDelay 1000000)
forkIO (forever $ queue hworker PrintB >> threadDelay 500000)
forever (threadDelay 1000000)
job hw PrintA =
let
State mvar = hworkerState hw
in do
v <- takeMVar mvar
if v == 0
then do
putMVar mvar 0
putStrLn "A" >> return Success
else do
putMVar mvar (v - 1)
error $ "Dying: " ++ show v

job _ PrintB =
putStrLn "B" >> return Success


main :: IO ()
main = do
mvar <- newMVar 3
hworker <- create "printer" (State mvar)
_ <- forkIO (worker hworker)
_ <- forkIO (monitor hworker)
_ <- forkIO (forever $ queue hworker PrintA >> threadDelay 1000000)
_ <- forkIO (forever $ queue hworker PrintB >> threadDelay 500000)
forever (threadDelay 1000000)
18 changes: 11 additions & 7 deletions hworker.cabal
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
name: hworker
version: 0.2.0
version: 0.3.0
synopsis: A reliable at-least-once job queue built on top of redis.
description: See README.
homepage: http://github.com/positiondev/hworker
Expand All @@ -13,7 +13,6 @@ cabal-version: >=1.10

library
exposed-modules: System.Hworker
other-modules: Data.Aeson.Helpers
build-depends: base >= 4.7 && < 5
, aeson
, hedis >= 0.6.5
Expand All @@ -22,16 +21,19 @@ library
, time >= 1.5
, attoparsec
, uuid >= 1.2.6
, mtl
, conduit
hs-source-dirs: src
default-language: Haskell2010
ghc-options: -Wall

Test-Suite hworker-test
type: exitcode-stdio-1.0
hs-source-dirs: src test
main-is: Spec.hs
other-modules: Data.Aeson.Helpers
, System.Hworker
type: exitcode-stdio-1.0
hs-source-dirs: src test
main-is: Spec.hs
other-modules: System.Hworker
ghc-options: -Wall
-fno-warn-unused-do-bind
build-depends: base >= 4.7 && < 5
, aeson
, hedis >= 0.6.5
Expand All @@ -43,3 +45,5 @@ Test-Suite hworker-test
, hspec >= 2
, hspec-contrib
, HUnit
, mtl
, conduit
20 changes: 0 additions & 20 deletions src/Data/Aeson/Helpers.hs

This file was deleted.

Loading