1313#include <linux/slab.h>
1414#include <linux/wait.h>
1515#include <linux/sched.h>
16+ #include <linux/cpu.h>
1617
1718#include "zcomp.h"
1819#include "zcomp_lzo.h"
1920#ifdef CONFIG_ZRAM_LZ4_COMPRESS
2021#include "zcomp_lz4.h"
2122#endif
2223
23- /*
24- * single zcomp_strm backend
25- */
26- struct zcomp_strm_single {
27- struct mutex strm_lock ;
28- struct zcomp_strm * zstrm ;
29- };
30-
31- /*
32- * multi zcomp_strm backend
33- */
34- struct zcomp_strm_multi {
35- /* protect strm list */
36- spinlock_t strm_lock ;
37- /* max possible number of zstrm streams */
38- int max_strm ;
39- /* number of available zstrm streams */
40- int avail_strm ;
41- /* list of available strms */
42- struct list_head idle_strm ;
43- wait_queue_head_t strm_wait ;
44- };
45-
4624static struct zcomp_backend * backends [] = {
4725 & zcomp_lzo ,
4826#ifdef CONFIG_ZRAM_LZ4_COMPRESS
@@ -93,188 +71,6 @@ static struct zcomp_strm *zcomp_strm_alloc(struct zcomp *comp, gfp_t flags)
9371 return zstrm ;
9472}
9573
96- /*
97- * get idle zcomp_strm or wait until other process release
98- * (zcomp_strm_release()) one for us
99- */
100- static struct zcomp_strm * zcomp_strm_multi_find (struct zcomp * comp )
101- {
102- struct zcomp_strm_multi * zs = comp -> stream ;
103- struct zcomp_strm * zstrm ;
104-
105- while (1 ) {
106- spin_lock (& zs -> strm_lock );
107- if (!list_empty (& zs -> idle_strm )) {
108- zstrm = list_entry (zs -> idle_strm .next ,
109- struct zcomp_strm , list );
110- list_del (& zstrm -> list );
111- spin_unlock (& zs -> strm_lock );
112- return zstrm ;
113- }
114- /* zstrm streams limit reached, wait for idle stream */
115- if (zs -> avail_strm >= zs -> max_strm ) {
116- spin_unlock (& zs -> strm_lock );
117- wait_event (zs -> strm_wait , !list_empty (& zs -> idle_strm ));
118- continue ;
119- }
120- /* allocate new zstrm stream */
121- zs -> avail_strm ++ ;
122- spin_unlock (& zs -> strm_lock );
123- /*
124- * This function can be called in swapout/fs write path
125- * so we can't use GFP_FS|IO. And it assumes we already
126- * have at least one stream in zram initialization so we
127- * don't do best effort to allocate more stream in here.
128- * A default stream will work well without further multiple
129- * streams. That's why we use NORETRY | NOWARN.
130- */
131- zstrm = zcomp_strm_alloc (comp , GFP_NOIO | __GFP_NORETRY |
132- __GFP_NOWARN );
133- if (!zstrm ) {
134- spin_lock (& zs -> strm_lock );
135- zs -> avail_strm -- ;
136- spin_unlock (& zs -> strm_lock );
137- wait_event (zs -> strm_wait , !list_empty (& zs -> idle_strm ));
138- continue ;
139- }
140- break ;
141- }
142- return zstrm ;
143- }
144-
145- /* add stream back to idle list and wake up waiter or free the stream */
146- static void zcomp_strm_multi_release (struct zcomp * comp , struct zcomp_strm * zstrm )
147- {
148- struct zcomp_strm_multi * zs = comp -> stream ;
149-
150- spin_lock (& zs -> strm_lock );
151- if (zs -> avail_strm <= zs -> max_strm ) {
152- list_add (& zstrm -> list , & zs -> idle_strm );
153- spin_unlock (& zs -> strm_lock );
154- wake_up (& zs -> strm_wait );
155- return ;
156- }
157-
158- zs -> avail_strm -- ;
159- spin_unlock (& zs -> strm_lock );
160- zcomp_strm_free (comp , zstrm );
161- }
162-
163- /* change max_strm limit */
164- static bool zcomp_strm_multi_set_max_streams (struct zcomp * comp , int num_strm )
165- {
166- struct zcomp_strm_multi * zs = comp -> stream ;
167- struct zcomp_strm * zstrm ;
168-
169- spin_lock (& zs -> strm_lock );
170- zs -> max_strm = num_strm ;
171- /*
172- * if user has lowered the limit and there are idle streams,
173- * immediately free as much streams (and memory) as we can.
174- */
175- while (zs -> avail_strm > num_strm && !list_empty (& zs -> idle_strm )) {
176- zstrm = list_entry (zs -> idle_strm .next ,
177- struct zcomp_strm , list );
178- list_del (& zstrm -> list );
179- zcomp_strm_free (comp , zstrm );
180- zs -> avail_strm -- ;
181- }
182- spin_unlock (& zs -> strm_lock );
183- return true;
184- }
185-
186- static void zcomp_strm_multi_destroy (struct zcomp * comp )
187- {
188- struct zcomp_strm_multi * zs = comp -> stream ;
189- struct zcomp_strm * zstrm ;
190-
191- while (!list_empty (& zs -> idle_strm )) {
192- zstrm = list_entry (zs -> idle_strm .next ,
193- struct zcomp_strm , list );
194- list_del (& zstrm -> list );
195- zcomp_strm_free (comp , zstrm );
196- }
197- kfree (zs );
198- }
199-
200- static int zcomp_strm_multi_create (struct zcomp * comp , int max_strm )
201- {
202- struct zcomp_strm * zstrm ;
203- struct zcomp_strm_multi * zs ;
204-
205- comp -> destroy = zcomp_strm_multi_destroy ;
206- comp -> strm_find = zcomp_strm_multi_find ;
207- comp -> strm_release = zcomp_strm_multi_release ;
208- comp -> set_max_streams = zcomp_strm_multi_set_max_streams ;
209- zs = kmalloc (sizeof (struct zcomp_strm_multi ), GFP_KERNEL );
210- if (!zs )
211- return - ENOMEM ;
212-
213- comp -> stream = zs ;
214- spin_lock_init (& zs -> strm_lock );
215- INIT_LIST_HEAD (& zs -> idle_strm );
216- init_waitqueue_head (& zs -> strm_wait );
217- zs -> max_strm = max_strm ;
218- zs -> avail_strm = 1 ;
219-
220- zstrm = zcomp_strm_alloc (comp , GFP_KERNEL );
221- if (!zstrm ) {
222- kfree (zs );
223- return - ENOMEM ;
224- }
225- list_add (& zstrm -> list , & zs -> idle_strm );
226- return 0 ;
227- }
228-
229- static struct zcomp_strm * zcomp_strm_single_find (struct zcomp * comp )
230- {
231- struct zcomp_strm_single * zs = comp -> stream ;
232- mutex_lock (& zs -> strm_lock );
233- return zs -> zstrm ;
234- }
235-
236- static void zcomp_strm_single_release (struct zcomp * comp ,
237- struct zcomp_strm * zstrm )
238- {
239- struct zcomp_strm_single * zs = comp -> stream ;
240- mutex_unlock (& zs -> strm_lock );
241- }
242-
243- static bool zcomp_strm_single_set_max_streams (struct zcomp * comp , int num_strm )
244- {
245- /* zcomp_strm_single support only max_comp_streams == 1 */
246- return false;
247- }
248-
249- static void zcomp_strm_single_destroy (struct zcomp * comp )
250- {
251- struct zcomp_strm_single * zs = comp -> stream ;
252- zcomp_strm_free (comp , zs -> zstrm );
253- kfree (zs );
254- }
255-
256- static int zcomp_strm_single_create (struct zcomp * comp )
257- {
258- struct zcomp_strm_single * zs ;
259-
260- comp -> destroy = zcomp_strm_single_destroy ;
261- comp -> strm_find = zcomp_strm_single_find ;
262- comp -> strm_release = zcomp_strm_single_release ;
263- comp -> set_max_streams = zcomp_strm_single_set_max_streams ;
264- zs = kmalloc (sizeof (struct zcomp_strm_single ), GFP_KERNEL );
265- if (!zs )
266- return - ENOMEM ;
267-
268- comp -> stream = zs ;
269- mutex_init (& zs -> strm_lock );
270- zs -> zstrm = zcomp_strm_alloc (comp , GFP_KERNEL );
271- if (!zs -> zstrm ) {
272- kfree (zs );
273- return - ENOMEM ;
274- }
275- return 0 ;
276- }
277-
27874/* show available compressors */
27975ssize_t zcomp_available_show (const char * comp , char * buf )
28076{
@@ -301,17 +97,17 @@ bool zcomp_available_algorithm(const char *comp)
30197
30298bool zcomp_set_max_streams (struct zcomp * comp , int num_strm )
30399{
304- return comp -> set_max_streams ( comp , num_strm ) ;
100+ return true ;
305101}
306102
307103struct zcomp_strm * zcomp_strm_find (struct zcomp * comp )
308104{
309- return comp -> strm_find ( comp );
105+ return * get_cpu_ptr ( comp -> stream );
310106}
311107
312108void zcomp_strm_release (struct zcomp * comp , struct zcomp_strm * zstrm )
313109{
314- comp -> strm_release ( comp , zstrm );
110+ put_cpu_ptr ( comp -> stream );
315111}
316112
317113int zcomp_compress (struct zcomp * comp , struct zcomp_strm * zstrm ,
@@ -327,9 +123,83 @@ int zcomp_decompress(struct zcomp *comp, const unsigned char *src,
327123 return comp -> backend -> decompress (src , src_len , dst );
328124}
329125
126+ static int __zcomp_cpu_notifier (struct zcomp * comp ,
127+ unsigned long action , unsigned long cpu )
128+ {
129+ struct zcomp_strm * zstrm ;
130+
131+ switch (action ) {
132+ case CPU_UP_PREPARE :
133+ if (WARN_ON (* per_cpu_ptr (comp -> stream , cpu )))
134+ break ;
135+ zstrm = zcomp_strm_alloc (comp , GFP_KERNEL );
136+ if (IS_ERR_OR_NULL (zstrm )) {
137+ pr_err ("Can't allocate a compression stream\n" );
138+ return NOTIFY_BAD ;
139+ }
140+ * per_cpu_ptr (comp -> stream , cpu ) = zstrm ;
141+ break ;
142+ case CPU_DEAD :
143+ case CPU_UP_CANCELED :
144+ zstrm = * per_cpu_ptr (comp -> stream , cpu );
145+ if (!IS_ERR_OR_NULL (zstrm ))
146+ zcomp_strm_free (comp , zstrm );
147+ * per_cpu_ptr (comp -> stream , cpu ) = NULL ;
148+ break ;
149+ default :
150+ break ;
151+ }
152+ return NOTIFY_OK ;
153+ }
154+
155+ static int zcomp_cpu_notifier (struct notifier_block * nb ,
156+ unsigned long action , void * pcpu )
157+ {
158+ unsigned long cpu = (unsigned long )pcpu ;
159+ struct zcomp * comp = container_of (nb , typeof (* comp ), notifier );
160+
161+ return __zcomp_cpu_notifier (comp , action , cpu );
162+ }
163+
164+ static int zcomp_init (struct zcomp * comp )
165+ {
166+ unsigned long cpu ;
167+ int ret ;
168+
169+ comp -> notifier .notifier_call = zcomp_cpu_notifier ;
170+
171+ comp -> stream = alloc_percpu (struct zcomp_strm * );
172+ if (!comp -> stream )
173+ return - ENOMEM ;
174+
175+ cpu_notifier_register_begin ();
176+ for_each_online_cpu (cpu ) {
177+ ret = __zcomp_cpu_notifier (comp , CPU_UP_PREPARE , cpu );
178+ if (ret == NOTIFY_BAD )
179+ goto cleanup ;
180+ }
181+ __register_cpu_notifier (& comp -> notifier );
182+ cpu_notifier_register_done ();
183+ return 0 ;
184+
185+ cleanup :
186+ for_each_online_cpu (cpu )
187+ __zcomp_cpu_notifier (comp , CPU_UP_CANCELED , cpu );
188+ cpu_notifier_register_done ();
189+ return - ENOMEM ;
190+ }
191+
330192void zcomp_destroy (struct zcomp * comp )
331193{
332- comp -> destroy (comp );
194+ unsigned long cpu ;
195+
196+ cpu_notifier_register_begin ();
197+ for_each_online_cpu (cpu )
198+ __zcomp_cpu_notifier (comp , CPU_UP_CANCELED , cpu );
199+ __unregister_cpu_notifier (& comp -> notifier );
200+ cpu_notifier_register_done ();
201+
202+ free_percpu (comp -> stream );
333203 kfree (comp );
334204}
335205
@@ -339,9 +209,9 @@ void zcomp_destroy(struct zcomp *comp)
339209 * backend pointer or ERR_PTR if things went bad. ERR_PTR(-EINVAL)
340210 * if requested algorithm is not supported, ERR_PTR(-ENOMEM) in
341211 * case of allocation error, or any other error potentially
342- * returned by functions zcomp_strm_{multi,single}_create .
212+ * returned by zcomp_init() .
343213 */
344- struct zcomp * zcomp_create (const char * compress , int max_strm )
214+ struct zcomp * zcomp_create (const char * compress )
345215{
346216 struct zcomp * comp ;
347217 struct zcomp_backend * backend ;
@@ -356,10 +226,7 @@ struct zcomp *zcomp_create(const char *compress, int max_strm)
356226 return ERR_PTR (- ENOMEM );
357227
358228 comp -> backend = backend ;
359- if (max_strm > 1 )
360- error = zcomp_strm_multi_create (comp , max_strm );
361- else
362- error = zcomp_strm_single_create (comp );
229+ error = zcomp_init (comp );
363230 if (error ) {
364231 kfree (comp );
365232 return ERR_PTR (error );
0 commit comments